Unable to diagnose

07:00:54.268 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:00:54.268 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskdgzdJQ/test-tmp
07:00:54.268 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:00:54.268 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskdgzdJQ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:00:54.270 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505"
  extra_tserver_flags: "--flush_threshold_mb=1"
  extra_tserver_flags: "--flush_threshold_secs=1"
  extra_tserver_flags: "--enable_rowset_compaction=false"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

07:00:54.589 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
07:00:54.590 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

07:00:54.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:54.593183 28579 mini_oidc.cc:152] Starting JWKS server
07:00:54.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:54.596556 28579 webserver.cc:460] Webserver started at http://127.0.0.1:44241/ using document root <none> and password file <none>
07:00:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:54.596940 28579 mini_oidc.cc:189] Starting OIDC Discovery server
07:00:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:54.598624 28579 webserver.cc:460] Webserver started at http://127.0.0.1:37125/ using document root <none> and password file <none>
07:00:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:00:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:00:54.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:00:54Z Disabled control of system clock
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:54.662709 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/logs
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data/info.pb
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.254:42503
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.254
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json with env {}
07:00:54.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.957026 28591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:00:54.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.957590 28591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:00:54.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.958081 28591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:00:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.986829 28591 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:00:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.987114 28591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:00:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.987316 28591 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:00:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.987516 28591 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:00:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.987709 28591 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:00:54.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:54.987870 28591 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44241/jwks.json
07:00:55.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:55.020565 28591 master_runner.cc:386] Master server non-default flags:
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.254:42503
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data/info.pb
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.254
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/logs
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:00:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:00:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:55.021840 28591 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:00:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:55.023386 28591 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:00:55.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:55.037427 28598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:55.037811 28597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:55.039880 28600 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.139921 28599 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:00:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.140026 28591 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:00:56.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.143757 28591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:00:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.146858 28591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:00:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.148300 28591 hybrid_clock.cc:648] HybridClock initialized: now 1714114856148238 us; error 75 us; skew 500 ppm
07:00:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.149116 28591 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:00:56.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.155783 28591 webserver.cc:460] Webserver started at http://127.27.232.254:43297/ using document root <none> and password file <none>
07:00:56.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.156749 28591 fs_manager.cc:359] Metadata directory not provided
07:00:56.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.156942 28591 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:00:56.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.157343 28591 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:00:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.162199 28591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data/instance:
07:00:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "549ffd67019748d2ad7365aea40cdec4"
07:00:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:56 on dist-test-slave-9fw8"
07:00:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.163239 28591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal/instance:
07:00:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "549ffd67019748d2ad7365aea40cdec4"
07:00:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:56 on dist-test-slave-9fw8"
07:00:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.170406 28591 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.003s
07:00:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.175992 28607 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:00:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.177109 28591 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
07:00:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.177407 28591 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal
07:00:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "549ffd67019748d2ad7365aea40cdec4"
07:00:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:56 on dist-test-slave-9fw8"
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.177719 28591 fs_report.cc:383] FS layout report
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data/data
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:00:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:00:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.222658 28591 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:00:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.224085 28591 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:00:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.224494 28591 kserver.cc:163] Server-wide thread pool size limit: 3276
07:00:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.293187 28591 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.254:42503
07:00:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.293272 28658 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.254:42503 every 8 connection(s)
07:00:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.295743 28591 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/data/info.pb
07:00:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.299393 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 28591
07:00:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.299954 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-0/wal/instance
07:00:56.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.301597 28659 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:00:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.305398 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal
07:00:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/logs
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data/info.pb
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.253:33513
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.253
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json with env {}
07:00:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.316826 28659 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } has no permanent_uuid. Determining permanent_uuid...
07:00:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.343434 28659 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } has no permanent_uuid. Determining permanent_uuid...
07:00:56.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.345546 28610 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.349366 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.398815 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 1
07:00:56.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.402624 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.470995 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 2
07:00:56.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.474714 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.566077 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 3
07:00:56.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.569782 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.608635 28661 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:00:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.609177 28661 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:00:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.609647 28661 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:00:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.639060 28661 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:00:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.639356 28661 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:00:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.639556 28661 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:00:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.639744 28661 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:00:56.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.639942 28661 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:00:56.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.640082 28661 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44241/jwks.json
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.672986 28661 master_runner.cc:386] Master server non-default flags:
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.253:33513
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data/info.pb
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.253
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:00:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/logs
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:00:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.674149 28661 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:00:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.675673 28661 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:00:56.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.689734 28670 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:56.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.713169 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 4
07:00:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.717758 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:57.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.017412 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 5
07:00:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:57.021894 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:57.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.569582 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 6
07:00:57.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:57.578552 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.253:33513: Network error: Client connection negotiation failed: client connection to 127.27.232.253:33513: connect: Connection refused (error 111)
07:00:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.690135 28671 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:56.691339 28661 server_base.cc:1034] running on GCE node
07:00:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:56.690613 28673 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:57.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.795517 28661 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:00:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.798524 28661 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:00:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.799926 28661 hybrid_clock.cc:648] HybridClock initialized: now 1714114857799888 us; error 55 us; skew 500 ppm
07:00:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.800696 28661 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:00:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.807333 28661 webserver.cc:460] Webserver started at http://127.27.232.253:36315/ using document root <none> and password file <none>
07:00:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.808264 28661 fs_manager.cc:359] Metadata directory not provided
07:00:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.808445 28661 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:00:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.808825 28661 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:00:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.813745 28661 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data/instance:
07:00:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b88b3fb38b4e4d7d97847e2034b362df"
07:00:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:57 on dist-test-slave-9fw8"
07:00:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.814749 28661 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal/instance:
07:00:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b88b3fb38b4e4d7d97847e2034b362df"
07:00:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:57 on dist-test-slave-9fw8"
07:00:57.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.822400 28661 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.004s
07:00:57.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.827639 28682 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:00:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.828630 28661 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.828902 28661 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b88b3fb38b4e4d7d97847e2034b362df"
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:57 on dist-test-slave-9fw8"
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.829180 28661 fs_report.cc:383] FS layout report
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data/data
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:00:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:00:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:00:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:00:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:00:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.881078 28661 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:00:57.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.882481 28661 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:00:57.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.882895 28661 kserver.cc:163] Server-wide thread pool size limit: 3276
07:00:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.949836 28661 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.253:33513
07:00:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.949934 28733 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.253:33513 every 8 connection(s)
07:00:57.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.952502 28661 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/data/info.pb
07:00:57.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.956517 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 28661
07:00:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.956982 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-1/wal/instance
07:00:57.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.958037 28734 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.960013 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/logs
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data/info.pb
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.252:45485
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.252
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json with env {}
07:00:57.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.971328 28734 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } has no permanent_uuid. Determining permanent_uuid...
07:00:57.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:57.994926 28734 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } has no permanent_uuid. Determining permanent_uuid...
07:00:58.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.008850 28734 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } has no permanent_uuid. Determining permanent_uuid...
07:00:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.011251 28683 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.015154 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.064574 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 1
07:00:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.068250 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.136742 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 2
07:00:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.140924 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.232295 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 3
07:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.236913 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.267092 28736 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:00:58.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.267630 28736 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:00:58.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.268147 28736 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:00:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.297310 28736 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:00:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.297605 28736 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:00:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.297798 28736 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:00:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.297994 28736 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:00:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.298184 28736 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:00:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.298336 28736 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44241/jwks.json
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.331739 28736 master_runner.cc:386] Master server non-default flags:
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.252:45485
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44241/jwks.json
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data/info.pb
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.252
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/logs
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:00:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:00:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:00:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:00:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.332947 28736 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:00:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.334517 28736 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:00:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.349597 28747 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.380256 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 4
07:00:58.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.384738 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.639194 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } attempt: 7
07:00:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.650897 28659 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } has no permanent_uuid. Determining permanent_uuid...
07:00:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.657435 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.684091 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 5
07:00:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.688530 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.715894 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 1
07:00:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.720286 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.801638 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 2
07:00:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.806249 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.891659 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 3
07:00:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.896715 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.037158 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 4
07:00:59.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.041788 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.235992 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 6
07:00:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.243126 28734 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:59.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.325228 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 5
07:00:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.329053 28659 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.232.252:45485: Network error: Client connection negotiation failed: client connection to 127.27.232.252:45485: connect: Connection refused (error 111)
07:00:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.349885 28746 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:58.350531 28749 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:00:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:58.352047 28736 server_base.cc:1034] running on GCE node
07:00:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.466972 28736 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:00:59.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.469542 28736 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:00:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.470882 28736 hybrid_clock.cc:648] HybridClock initialized: now 1714114859470858 us; error 41 us; skew 500 ppm
07:00:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.471653 28736 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:00:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.482661 28736 webserver.cc:460] Webserver started at http://127.27.232.252:44991/ using document root <none> and password file <none>
07:00:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.483533 28736 fs_manager.cc:359] Metadata directory not provided
07:00:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.483713 28736 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:00:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.484127 28736 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:00:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.488859 28736 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data/instance:
07:00:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94255d7cbb8b42a1bf42d16c4892a92e"
07:00:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:59 on dist-test-slave-9fw8"
07:00:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.489871 28736 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal/instance:
07:00:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94255d7cbb8b42a1bf42d16c4892a92e"
07:00:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:59 on dist-test-slave-9fw8"
07:00:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.496603 28736 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.001s
07:00:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.501705 28761 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:00:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.502696 28736 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.001s
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.502972 28736 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94255d7cbb8b42a1bf42d16c4892a92e"
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:00:59 on dist-test-slave-9fw8"
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.503268 28736 fs_report.cc:383] FS layout report
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data/data
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:00:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:00:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.552781 28736 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:00:59.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.554124 28736 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:00:59.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.554565 28736 kserver.cc:163] Server-wide thread pool size limit: 3276
07:00:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.621271 28736 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.252:45485
07:00:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.621416 28812 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.252:45485 every 8 connection(s)
07:00:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.623878 28736 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/data/info.pb
07:00:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.628965 28813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:00:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.630429 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 28736
07:00:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.630928 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/master-2/wal/instance
07:00:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.645682 28813 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } has no permanent_uuid. Determining permanent_uuid...
07:00:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.667460 28813 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } has no permanent_uuid. Determining permanent_uuid...
07:00:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.676905 28813 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } has no permanent_uuid. Determining permanent_uuid...
07:00:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.691779 28813 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: Bootstrap starting.
07:00:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.697388 28813 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: Neither blocks nor log segments found. Creating new log.
07:00:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.699682 28813 log.cc:824] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: Log is configured to *not* fsync() on all Append() calls
07:00:59.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.703699 28813 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: No bootstrap required, opened a new log
07:00:59.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.724936 28813 raft_consensus.cc:348] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:00:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.725839 28813 raft_consensus.cc:374] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:00:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.726152 28813 raft_consensus.cc:729] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 94255d7cbb8b42a1bf42d16c4892a92e, State: Initialized, Role: FOLLOWER
07:00:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.727028 28813 consensus_queue.cc:260] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:00:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.729869 28822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:00:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.730757 28822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: This master's current role is: FOLLOWER
07:00:59.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.731590 28813 sys_catalog.cc:564] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: configured and running, proceeding with master startup.
07:00:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.756870 28833 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:00:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.757216 28833 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:00:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.753818 28736 master_runner.cc:186] Error getting master registration for 127.27.232.254:42503: OK, instance_id { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" instance_seqno: 1714114856274573 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:00:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.776502 28736 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:00:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.809305 28767 tablet.cc:2369] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:00:59.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.881438 28659 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 6
07:00:59.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.897660 28659 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: Bootstrap starting.
07:00:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.903942 28659 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: Neither blocks nor log segments found. Creating new log.
07:00:59.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.906008 28659 log.cc:824] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: Log is configured to *not* fsync() on all Append() calls
07:00:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.910365 28659 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: No bootstrap required, opened a new log
07:00:59.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.932507 28659 raft_consensus.cc:348] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:00:59.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.933403 28659 raft_consensus.cc:374] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:00:59.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.933637 28659 raft_consensus.cc:729] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 549ffd67019748d2ad7365aea40cdec4, State: Initialized, Role: FOLLOWER
07:00:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.934345 28659 consensus_queue.cc:260] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:00:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.937335 28837 sys_catalog.cc:455] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:00:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.938194 28837 sys_catalog.cc:458] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: This master's current role is: FOLLOWER
07:00:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.939108 28659 sys_catalog.cc:564] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: configured and running, proceeding with master startup.
07:00:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.983343 28613 tablet.cc:2369] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:00:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.985822 28848 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:00:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:00:59.986279 28848 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:01:00.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:59.971879 28591 master_runner.cc:186] Error getting master registration for 127.27.232.253:33513: OK, instance_id { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" instance_seqno: 1714114857931613 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:01:00.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.011381 28591 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:01:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.204967 28822 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.205555 28822 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:01:00.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.208056 28822 leader_election.cc:288] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 549ffd67019748d2ad7365aea40cdec4 (127.27.232.254:42503), b88b3fb38b4e4d7d97847e2034b362df (127.27.232.253:33513)
07:01:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.209532 28634 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "549ffd67019748d2ad7365aea40cdec4" is_pre_election: true
07:01:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.210445 28634 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 94255d7cbb8b42a1bf42d16c4892a92e in term 0.
07:01:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.211817 28764 leader_election.cc:302] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 549ffd67019748d2ad7365aea40cdec4, 94255d7cbb8b42a1bf42d16c4892a92e; no voters: 
07:01:00.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.212885 28822 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Leader pre-election won for term 1
07:01:00.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.213305 28822 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:00.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.213716 28822 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 0 FOLLOWER]: Advancing to term 1
07:01:00.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.220162 28822 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:01:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.222869 28634 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "549ffd67019748d2ad7365aea40cdec4"
07:01:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.223424 28634 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 0 FOLLOWER]: Advancing to term 1
07:01:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.227347 28822 leader_election.cc:288] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [CANDIDATE]: Term 1 election: Requested vote from peers 549ffd67019748d2ad7365aea40cdec4 (127.27.232.254:42503), b88b3fb38b4e4d7d97847e2034b362df (127.27.232.253:33513)
07:01:00.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.233024 28634 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 94255d7cbb8b42a1bf42d16c4892a92e in term 1.
07:01:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.240514 28764 leader_election.cc:302] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 549ffd67019748d2ad7365aea40cdec4, 94255d7cbb8b42a1bf42d16c4892a92e; no voters: 
07:01:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.241529 28822 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 1 FOLLOWER]: Leader election won for term 1
07:01:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.244442 28822 raft_consensus.cc:686] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [term 1 LEADER]: Becoming Leader. State: Replica: 94255d7cbb8b42a1bf42d16c4892a92e, State: Running, Role: LEADER
07:01:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.246346 28822 consensus_queue.cc:237] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:01:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.261461 28850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 94255d7cbb8b42a1bf42d16c4892a92e. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.262229 28850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: This master's current role is: LEADER
07:01:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.272948 28852 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:01:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.280571 28852 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:01:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.297986 28634 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [term 1 FOLLOWER]: Refusing update from remote peer 94255d7cbb8b42a1bf42d16c4892a92e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:00.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.299729 28822 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [LEADER]: Connected to new peer: Peer: permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:00.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.303725 28734 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } attempt: 7
07:01:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.339908 28837 sys_catalog.cc:455] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 94255d7cbb8b42a1bf42d16c4892a92e. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.340689 28837 sys_catalog.cc:458] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: This master's current role is: FOLLOWER
07:01:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.354079 28853 mvcc.cc:205] Tried to move back new op lower bound from 7021014467763535872 to 7021014467589603328. Current Snapshot: MvccSnapshot[applied={T|T < 7021014467763535872}]
07:01:00.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.365125 28850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.365866 28850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: This master's current role is: LEADER
07:01:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.367254 28852 catalog_manager.cc:1347] Generated new cluster ID: e725548f94f345538b815f8fb2b746a6
07:01:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.367547 28852 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:01:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.368474 28734 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: Bootstrap starting.
07:01:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.372411 28837 sys_catalog.cc:455] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.373114 28837 sys_catalog.cc:458] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4 [sys.catalog]: This master's current role is: FOLLOWER
07:01:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.386310 28734 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: Neither blocks nor log segments found. Creating new log.
07:01:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.388641 28734 log.cc:824] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: Log is configured to *not* fsync() on all Append() calls
07:01:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.398037 28734 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: No bootstrap required, opened a new log
07:01:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.415657 28852 catalog_manager.cc:1370] Generated new certificate authority record
07:01:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.417690 28852 catalog_manager.cc:1504] Loading token signing keys...
07:01:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.453262 28852 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e: Generated new TSK 0
07:01:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.454478 28852 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.473134 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/logs
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data/info.pb
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.193:0
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.193
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.193
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:01:00.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.491209 28734 raft_consensus.cc:348] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:01:00.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.492098 28734 raft_consensus.cc:374] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:00.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.492422 28734 raft_consensus.cc:729] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b88b3fb38b4e4d7d97847e2034b362df, State: Initialized, Role: FOLLOWER
07:01:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.493322 28734 consensus_queue.cc:260] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } }
07:01:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.497674 28861 sys_catalog.cc:455] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.499675 28861 sys_catalog.cc:458] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: This master's current role is: FOLLOWER
07:01:00.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.513186 28734 sys_catalog.cc:564] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: configured and running, proceeding with master startup.
07:01:00.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.513783 28707 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [term 0 FOLLOWER]: Advancing to term 1
07:01:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.519618 28708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b88b3fb38b4e4d7d97847e2034b362df"
07:01:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.522220 28709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b88b3fb38b4e4d7d97847e2034b362df" is_pre_election: true
07:01:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.526614 28707 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [term 1 FOLLOWER]: Refusing update from remote peer 94255d7cbb8b42a1bf42d16c4892a92e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.528481 28822 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:00.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.583434 28861 sys_catalog.cc:455] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 94255d7cbb8b42a1bf42d16c4892a92e. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.584429 28861 sys_catalog.cc:458] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: This master's current role is: FOLLOWER
07:01:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.585608 28867 sys_catalog.cc:455] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.586814 28867 sys_catalog.cc:458] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df [sys.catalog]: This master's current role is: FOLLOWER
07:01:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.607452 28877 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:01:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.911569 28822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "549ffd67019748d2ad7365aea40cdec4" member_type: VOTER last_known_addr { host: "127.27.232.254" port: 42503 } } peers { permanent_uuid: "b88b3fb38b4e4d7d97847e2034b362df" member_type: VOTER last_known_addr { host: "127.27.232.253" port: 33513 } } peers { permanent_uuid: "94255d7cbb8b42a1bf42d16c4892a92e" member_type: VOTER last_known_addr { host: "127.27.232.252" port: 45485 } } }
07:01:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.916065 28822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94255d7cbb8b42a1bf42d16c4892a92e [sys.catalog]: This master's current role is: LEADER
07:01:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.992305 28848 catalog_manager.cc:1259] Loaded cluster ID: e725548f94f345538b815f8fb2b746a6
07:01:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:00.992830 28848 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: loading cluster ID for follower catalog manager: success
07:01:00.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:00.999280 28860 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.000099 28860 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.000391 28848 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: acquiring CA information for follower catalog manager: success
07:01:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.000888 28860 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:01:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.001554 28860 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.017560 28848 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 549ffd67019748d2ad7365aea40cdec4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:01:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.038702 28860 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.039410 28860 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:01:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.039654 28860 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:01:01.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.040326 28860 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.232.193
07:01:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.075466 28860 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.193:0
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data/info.pb
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.193
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.193
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/logs
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:01:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:01:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.076961 28860 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.079234 28860 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.102082 28883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:01.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.618520 28877 catalog_manager.cc:1259] Loaded cluster ID: e725548f94f345538b815f8fb2b746a6
07:01:01.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.618788 28877 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: loading cluster ID for follower catalog manager: success
07:01:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.637331 28877 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: acquiring CA information for follower catalog manager: success
07:01:01.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:01.649953 28877 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b88b3fb38b4e4d7d97847e2034b362df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:01:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:02.496605 28882 debug-util.cc:398] Leaking SignalData structure 0x7b0800014ea0 after lost signal to thread 28860
07:01:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:03.425364 28860 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.323s	user 0.534s	sys 0.911s
07:01:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:01.104416 28884 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:03.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:03.425726 28860 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.324s	user 0.534s	sys 0.911s
07:01:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:03.426611 28886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.426621 28860 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:01:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:03.424923 28885 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2320 milliseconds
07:01:03.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.431450 28860 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.434242 28860 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.435649 28860 hybrid_clock.cc:648] HybridClock initialized: now 1714114863435591 us; error 73 us; skew 500 ppm
07:01:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.436532 28860 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.443938 28860 webserver.cc:460] Webserver started at http://127.27.232.193:38481/ using document root <none> and password file <none>
07:01:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.444839 28860 fs_manager.cc:359] Metadata directory not provided
07:01:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.445055 28860 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.445513 28860 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.450237 28860 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data/instance:
07:01:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5d236d712aa4f3f88093b1d06726d10"
07:01:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:03 on dist-test-slave-9fw8"
07:01:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.451351 28860 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal/instance:
07:01:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5d236d712aa4f3f88093b1d06726d10"
07:01:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:03 on dist-test-slave-9fw8"
07:01:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.458868 28860 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
07:01:03.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.464733 28894 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.465790 28860 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.466101 28860 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5d236d712aa4f3f88093b1d06726d10"
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:03 on dist-test-slave-9fw8"
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.466421 28860 fs_report.cc:383] FS layout report
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data/data
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.525712 28860 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:01:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.527170 28860 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.527639 28860 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.530656 28860 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:01:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.535712 28860 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.536008 28860 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
07:01:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.536305 28860 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.536477 28860 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
07:01:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.741082 28860 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.193:33459
07:01:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.741304 29007 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.193:33459 every 8 connection(s)
07:01:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.743970 28860 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/data/info.pb
07:01:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.746596 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 28860
07:01:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.747067 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-0/wal/instance
07:01:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.788287 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal
07:01:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data
07:01:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/logs
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data/info.pb
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.194:0
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.194
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.194
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:01:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.821053 29009 heartbeater.cc:347] Connected to a master server at 127.27.232.253:33513
07:01:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.821492 29009 heartbeater.cc:464] Registering TS with master...
07:01:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.822744 29009 heartbeater.cc:511] Master 127.27.232.253:33513 requested a full tablet report, sending...
07:01:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.826460 28699 ts_manager.cc:194] Registered new tserver with Master: d5d236d712aa4f3f88093b1d06726d10 (127.27.232.193:33459)
07:01:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.835124 29008 heartbeater.cc:347] Connected to a master server at 127.27.232.252:45485
07:01:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.835446 29008 heartbeater.cc:464] Registering TS with master...
07:01:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.836287 29008 heartbeater.cc:511] Master 127.27.232.252:45485 requested a full tablet report, sending...
07:01:03.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.838466 28778 ts_manager.cc:194] Registered new tserver with Master: d5d236d712aa4f3f88093b1d06726d10 (127.27.232.193:33459)
07:01:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.840488 28778 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.232.193:49179
07:01:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.853740 29014 heartbeater.cc:347] Connected to a master server at 127.27.232.254:42503
07:01:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.854131 29014 heartbeater.cc:464] Registering TS with master...
07:01:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.854888 29014 heartbeater.cc:511] Master 127.27.232.254:42503 requested a full tablet report, sending...
07:01:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:03.858213 28624 ts_manager.cc:194] Registered new tserver with Master: d5d236d712aa4f3f88093b1d06726d10 (127.27.232.193:33459)
07:01:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.181519 29016 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.182053 29016 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.182560 29016 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:01:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.183146 29016 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.215265 29016 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.216037 29016 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:01:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.216277 29016 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:01:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.216912 29016 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.232.194
07:01:04.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:04.250794 29016 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:04.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:04.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:04.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:04.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.194:0
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data/info.pb
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.194
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.194
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/logs
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:01:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:04.252275 29016 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:04.254133 29016 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:04.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.272634 29025 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:04.843866 29008 heartbeater.cc:503] Master 127.27.232.252:45485 was elected leader, sending a full tablet report...
07:01:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.276204 29027 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:04.277855 29016 server_base.cc:1034] running on GCE node
07:01:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:04.276454 29024 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.406534 29016 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:05.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.408849 29016 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:05.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.410195 29016 hybrid_clock.cc:648] HybridClock initialized: now 1714114865410167 us; error 48 us; skew 500 ppm
07:01:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.410988 29016 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.421938 29016 webserver.cc:460] Webserver started at http://127.27.232.194:42881/ using document root <none> and password file <none>
07:01:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.422796 29016 fs_manager.cc:359] Metadata directory not provided
07:01:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.422977 29016 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.423419 29016 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.428546 29016 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data/instance:
07:01:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "873d5cd21db94f939e82ce1c854ff24c"
07:01:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:05 on dist-test-slave-9fw8"
07:01:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.429550 29016 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal/instance:
07:01:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "873d5cd21db94f939e82ce1c854ff24c"
07:01:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:05 on dist-test-slave-9fw8"
07:01:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.436792 29016 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
07:01:05.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.442625 29036 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:01:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.443650 29016 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.444000 29016 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "873d5cd21db94f939e82ce1c854ff24c"
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:05 on dist-test-slave-9fw8"
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.444352 29016 fs_report.cc:383] FS layout report
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal
07:01:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data/data
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:05.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.504761 29016 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:01:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.506291 29016 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:05.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.506763 29016 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.509251 29016 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:01:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.513521 29016 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.513738 29016 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
07:01:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.514027 29016 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.514241 29016 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
07:01:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.652458 29016 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.194:41829
07:01:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.652550 29149 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.194:41829 every 8 connection(s)
07:01:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.655256 29016 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/data/info.pb
07:01:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.659163 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 29016
07:01:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.659965 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-1/wal/instance
07:01:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.684214 28579 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu
07:01:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal
07:01:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data
07:01:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/logs
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data/info.pb
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.195:0
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.195
07:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.195
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:01:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.702260 29150 heartbeater.cc:347] Connected to a master server at 127.27.232.252:45485
07:01:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.702790 29150 heartbeater.cc:464] Registering TS with master...
07:01:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.704278 29150 heartbeater.cc:511] Master 127.27.232.252:45485 requested a full tablet report, sending...
07:01:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.707340 28778 ts_manager.cc:194] Registered new tserver with Master: 873d5cd21db94f939e82ce1c854ff24c (127.27.232.194:41829)
07:01:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.709342 28778 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.232.194:48583
07:01:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.715994 29151 heartbeater.cc:347] Connected to a master server at 127.27.232.253:33513
07:01:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.716290 29151 heartbeater.cc:464] Registering TS with master...
07:01:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.716830 29151 heartbeater.cc:511] Master 127.27.232.253:33513 requested a full tablet report, sending...
07:01:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.717213 29152 heartbeater.cc:347] Connected to a master server at 127.27.232.254:42503
07:01:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.717635 29152 heartbeater.cc:464] Registering TS with master...
07:01:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.718369 29152 heartbeater.cc:511] Master 127.27.232.254:42503 requested a full tablet report, sending...
07:01:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.719056 28699 ts_manager.cc:194] Registered new tserver with Master: 873d5cd21db94f939e82ce1c854ff24c (127.27.232.194:41829)
07:01:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:05.721506 28624 ts_manager.cc:194] Registered new tserver with Master: 873d5cd21db94f939e82ce1c854ff24c (127.27.232.194:41829)
07:01:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.007326 29160 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.007804 29160 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.008224 29160 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:01:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.008553 29160 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.037587 29160 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.038163 29160 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:01:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.038342 29160 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:01:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.038779 29160 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.232.195
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.071168 29160 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.232.212:37555
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.232.195:0
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data/info.pb
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.232.195
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.232.254:42503,127.27.232.253:33513,127.27.232.252:45485
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.232.195
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/logs
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:01:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.072315 29160 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.073802 29160 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.088982 29166 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.713465 29150 heartbeater.cc:503] Master 127.27.232.252:45485 was elected leader, sending a full tablet report...
07:01:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.089180 29167 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.090174 29169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.091269 29160 server_base.cc:1034] running on GCE node
07:01:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.187566 29160 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.189833 29160 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.191210 29160 hybrid_clock.cc:648] HybridClock initialized: now 1714114867191165 us; error 53 us; skew 500 ppm
07:01:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.192057 29160 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.198089 29160 webserver.cc:460] Webserver started at http://127.27.232.195:41365/ using document root <none> and password file <none>
07:01:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.199012 29160 fs_manager.cc:359] Metadata directory not provided
07:01:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.199226 29160 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.199744 29160 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.204608 29160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data/instance:
07:01:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b5341cc970c4a9b8909d7fce32b878f"
07:01:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-9fw8"
07:01:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.205708 29160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal/instance:
07:01:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b5341cc970c4a9b8909d7fce32b878f"
07:01:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-9fw8"
07:01:07.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.212641 29160 fs_manager.cc:691] Time spent creating directory manager: real 0.006s	user 0.008s	sys 0.000s
07:01:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.218116 29177 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
07:01:07.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.219101 29160 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.006s	sys 0.000s
07:01:07.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.219424 29160 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data,/tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal
07:01:07.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b5341cc970c4a9b8909d7fce32b878f"
07:01:07.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-9fw8"
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.219760 29160 fs_report.cc:383] FS layout report
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data/data
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.283432 29160 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
07:01:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.284973 29160 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.285452 29160 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.288141 29160 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:01:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.292445 29160 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.292656 29160 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
07:01:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.292953 29160 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.293134 29160 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
07:01:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.437305 29160 rpc_server.cc:307] RPC server started. Bound to: 127.27.232.195:38145
07:01:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.437379 29289 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.232.195:38145 every 8 connection(s)
07:01:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.439973 29160 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/data/info.pb
07:01:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.445989 28579 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu as pid 29160
07:01:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.446580 28579 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskdgzdJQ/test-tmp/mini-kudu-cluster3789528099145725505/ts-2/wal/instance
07:01:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.492221 29291 heartbeater.cc:347] Connected to a master server at 127.27.232.253:33513
07:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.492753 29291 heartbeater.cc:464] Registering TS with master...
07:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.493054 29290 heartbeater.cc:347] Connected to a master server at 127.27.232.252:45485
07:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.493397 29290 heartbeater.cc:464] Registering TS with master...
07:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.493836 29291 heartbeater.cc:511] Master 127.27.232.253:33513 requested a full tablet report, sending...
07:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.494526 29290 heartbeater.cc:511] Master 127.27.232.252:45485 requested a full tablet report, sending...
07:01:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.496924 28778 ts_manager.cc:194] Registered new tserver with Master: 8b5341cc970c4a9b8909d7fce32b878f (127.27.232.195:38145)
07:01:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.497126 28698 ts_manager.cc:194] Registered new tserver with Master: 8b5341cc970c4a9b8909d7fce32b878f (127.27.232.195:38145)
07:01:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.498453 28778 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.232.195:39543
07:01:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.501332 29292 heartbeater.cc:347] Connected to a master server at 127.27.232.254:42503
07:01:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.501639 29292 heartbeater.cc:464] Registering TS with master...
07:01:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.502228 29292 heartbeater.cc:511] Master 127.27.232.254:42503 requested a full tablet report, sending...
07:01:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.504227 28624 ts_manager.cc:194] Registered new tserver with Master: 8b5341cc970c4a9b8909d7fce32b878f (127.27.232.195:38145)
07:01:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.520236 28579 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:01:07.530 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
07:01:07.530 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

07:01:07.531 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.27.232.254"
      port: 42503
    }
    bound_http_address {
      host: "127.27.232.254"
      port: 43297
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.27.232.253"
      port: 33513
    }
    bound_http_address {
      host: "127.27.232.253"
      port: 36315
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.27.232.252"
      port: 45485
    }
    bound_http_address {
      host: "127.27.232.252"
      port: 44991
    }
  }
}

07:01:07.532 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

07:01:07.533 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.27.232.193"
      port: 33459
    }
    bound_http_address {
      host: "127.27.232.193"
      port: 38481
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.27.232.194"
      port: 41829
    }
    bound_http_address {
      host: "127.27.232.194"
      port: 42881
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.27.232.195"
      port: 38145
    }
    bound_http_address {
      host: "127.27.232.195"
      port: 41365
    }
  }
}

07:01:07.533 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:01:07.535 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
07:01:07.537 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
07:01:07.538 [INFO - main] (Logging.scala:57) ==============================================================
07:01:07.538 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
07:01:07.538 [INFO - main] (Logging.scala:57) ==============================================================
07:01:07.538 [INFO - main] (Logging.scala:57) Submitted application: test
07:01:07.539 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
07:01:07.540 [INFO - main] (Logging.scala:57) Limiting resource is cpu
07:01:07.540 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
07:01:07.541 [INFO - main] (Logging.scala:57) Changing view acls to: slave
07:01:07.542 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
07:01:07.542 [INFO - main] (Logging.scala:57) Changing view acls groups to: 
07:01:07.542 [INFO - main] (Logging.scala:57) Changing modify acls groups to: 
07:01:07.542 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(slave); groups with view permissions: Set(); users  with modify permissions: Set(slave); groups with modify permissions: Set()
07:01:07.562 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 43849.
07:01:07.568 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
07:01:07.569 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
07:01:07.570 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
07:01:07.570 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
07:01:07.570 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
07:01:07.571 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-d8abbfda-8094-49fa-82f7-ecd9c454e1af
07:01:07.572 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
07:01:07.573 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
07:01:07.599 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-9fw8.c.gcp-upstream.internal
07:01:07.603 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 39879.
07:01:07.603 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-9fw8.c.gcp-upstream.internal:39879
07:01:07.603 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
07:01:07.604 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-9fw8.c.gcp-upstream.internal, 39879, None)
07:01:07.604 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-9fw8.c.gcp-upstream.internal:39879 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-9fw8.c.gcp-upstream.internal, 39879, None)
07:01:07.605 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-9fw8.c.gcp-upstream.internal, 39879, None)
07:01:07.605 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-9fw8.c.gcp-upstream.internal, 39879, None)
07:01:07.614 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
07:01:07.626 [DEBUG - main] (Connection.java:748) [peer master-127.27.232.252:45485(127.27.232.252:45485)] connecting to peer
07:01:07.628 [DEBUG - main] (Connection.java:748) [peer master-127.27.232.254:42503(127.27.232.254:42503)] connecting to peer
07:01:07.629 [DEBUG - main] (Connection.java:748) [peer master-127.27.232.253:33513(127.27.232.253:33513)] connecting to peer
07:01:07.632 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.232.252:45485(127.27.232.252:45485)] Successfully connected to peer
07:01:07.634 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.232.252
07:01:07.643 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.232.253:33513(127.27.232.253:33513)] Successfully connected to peer
07:01:07.644 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.232.254:42503(127.27.232.254:42503)] Successfully connected to peer
07:01:07.649 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.232.254
07:01:07.655 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:01:07.657 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.232.253
07:01:07.661 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xaba2c9fc, L:/127.0.0.1:53482 - R:/127.27.232.252:45485] using SASL/PLAIN
07:01:07.665 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:01:07.671 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:01:07.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:01:07.675 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc7b60983, L:/127.0.0.1:58112 - R:/127.27.232.253:33513] using SASL/PLAIN
07:01:07.675 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x344d194e, L:/127.0.0.1:57526 - R:/127.27.232.254:42503] using SASL/PLAIN
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.679714 28778 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53482:
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c1_i"
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c2_s"
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c3_double"
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DOUBLE
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c4_long"
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c5_bool"
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c6_short"
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT16
07:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c7_float"
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: FLOAT
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c8_binary"
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BINARY
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c9_unixtime_micros"
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: UNIXTIME_MICROS
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c10_byte"
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c11_decimal32"
07:01:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL32
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 9
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c12_decimal64"
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL64
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 18
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c13_decimal128"
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL128
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 38
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c14_varchar"
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: VARCHAR
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       length: 65535
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c15_date"
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DATE
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
07:01:07.720 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 498c2ea76ae249288b7ddbc5567665e8
07:01:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.762341 29225 tablet_service.cc:1467] Processing CreateTablet for tablet a0c99eb2ea1b4642920d048cd5326cf0 (DEFAULT_TABLE table=test [id=498c2ea76ae249288b7ddbc5567665e8]), partition=RANGE (key) PARTITION VALUES < 50
07:01:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.764214 29225 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a0c99eb2ea1b4642920d048cd5326cf0. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:07.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.777546 28942 tablet_service.cc:1467] Processing CreateTablet for tablet 8cf7c867c20f4638b9590c022978414e (DEFAULT_TABLE table=test [id=498c2ea76ae249288b7ddbc5567665e8]), partition=RANGE (key) PARTITION 50 <= VALUES
07:01:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.779310 28942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8cf7c867c20f4638b9590c022978414e. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.786597 29345 tablet_bootstrap.cc:492] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: Bootstrap starting.
07:01:07.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.793608 29345 tablet_bootstrap.cc:654] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: Neither blocks nor log segments found. Creating new log.
07:01:07.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.796056 29345 log.cc:824] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: Log is configured to *not* fsync() on all Append() calls
07:01:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.802078 29345 tablet_bootstrap.cc:492] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: No bootstrap required, opened a new log
07:01:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.802611 29345 ts_tablet_manager.cc:1367] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: Time spent bootstrapping tablet: real 0.016s	user 0.004s	sys 0.009s
07:01:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.803748 29347 tablet_bootstrap.cc:492] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: Bootstrap starting.
07:01:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.808751 29347 tablet_bootstrap.cc:654] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: Neither blocks nor log segments found. Creating new log.
07:01:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.810945 29347 log.cc:824] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: Log is configured to *not* fsync() on all Append() calls
07:01:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.815464 29347 tablet_bootstrap.cc:492] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: No bootstrap required, opened a new log
07:01:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.815899 29347 ts_tablet_manager.cc:1367] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: Time spent bootstrapping tablet: real 0.012s	user 0.008s	sys 0.003s
07:01:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.824407 29345 raft_consensus.cc:348] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8b5341cc970c4a9b8909d7fce32b878f" member_type: VOTER last_known_addr { host: "127.27.232.195" port: 38145 } }
07:01:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.825142 29345 raft_consensus.cc:374] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.825429 29345 raft_consensus.cc:729] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b5341cc970c4a9b8909d7fce32b878f, State: Initialized, Role: FOLLOWER
07:01:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.826081 29345 consensus_queue.cc:260] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8b5341cc970c4a9b8909d7fce32b878f" member_type: VOTER last_known_addr { host: "127.27.232.195" port: 38145 } }
07:01:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.826565 29345 raft_consensus.cc:388] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:01:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.826786 29345 raft_consensus.cc:482] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:01:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.827107 29345 raft_consensus.cc:3037] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 0 FOLLOWER]: Advancing to term 1
07:01:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.830947 29345 raft_consensus.cc:504] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8b5341cc970c4a9b8909d7fce32b878f" member_type: VOTER last_known_addr { host: "127.27.232.195" port: 38145 } }
07:01:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.831555 29345 leader_election.cc:302] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8b5341cc970c4a9b8909d7fce32b878f; no voters: 
07:01:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.833770 29345 leader_election.cc:288] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [CANDIDATE]: Term 1 election: Requested vote from peers 
07:01:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.834172 29349 raft_consensus.cc:2781] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 1 FOLLOWER]: Leader election won for term 1
07:01:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.837435 29349 raft_consensus.cc:686] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [term 1 LEADER]: Becoming Leader. State: Replica: 8b5341cc970c4a9b8909d7fce32b878f, State: Running, Role: LEADER
07:01:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.838282 29349 consensus_queue.cc:237] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8b5341cc970c4a9b8909d7fce32b878f" member_type: VOTER last_known_addr { host: "127.27.232.195" port: 38145 } }
07:01:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.838775 29290 heartbeater.cc:503] Master 127.27.232.252:45485 was elected leader, sending a full tablet report...
07:01:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.839932 29347 raft_consensus.cc:348] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d5d236d712aa4f3f88093b1d06726d10" member_type: VOTER last_known_addr { host: "127.27.232.193" port: 33459 } }
07:01:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.841063 29347 raft_consensus.cc:374] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.841405 29347 raft_consensus.cc:729] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5d236d712aa4f3f88093b1d06726d10, State: Initialized, Role: FOLLOWER
07:01:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.842331 29347 consensus_queue.cc:260] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d5d236d712aa4f3f88093b1d06726d10" member_type: VOTER last_known_addr { host: "127.27.232.193" port: 33459 } }
07:01:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.843690 29347 raft_consensus.cc:388] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:01:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.844175 29347 raft_consensus.cc:482] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:01:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.844579 29347 raft_consensus.cc:3037] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 0 FOLLOWER]: Advancing to term 1
07:01:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.850416 29345 ts_tablet_manager.cc:1398] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f: Time spent starting tablet: real 0.047s	user 0.033s	sys 0.008s
07:01:07.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.851006 29347 raft_consensus.cc:504] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d5d236d712aa4f3f88093b1d06726d10" member_type: VOTER last_known_addr { host: "127.27.232.193" port: 33459 } }
07:01:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.851979 29347 leader_election.cc:302] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d5d236d712aa4f3f88093b1d06726d10; no voters: 
07:01:07.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.859755 28777 catalog_manager.cc:5484] T a0c99eb2ea1b4642920d048cd5326cf0 P 8b5341cc970c4a9b8909d7fce32b878f reported cstate change: term changed from 0 to 1, leader changed from <none> to 8b5341cc970c4a9b8909d7fce32b878f (127.27.232.195). New cstate: current_term: 1 leader_uuid: "8b5341cc970c4a9b8909d7fce32b878f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8b5341cc970c4a9b8909d7fce32b878f" member_type: VOTER last_known_addr { host: "127.27.232.195" port: 38145 } health_report { overall_health: HEALTHY } } }
07:01:07.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.862231 29353 raft_consensus.cc:2781] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 1 FOLLOWER]: Leader election won for term 1
07:01:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.864008 29347 leader_election.cc:288] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [CANDIDATE]: Term 1 election: Requested vote from peers 
07:01:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.866463 29353 raft_consensus.cc:686] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [term 1 LEADER]: Becoming Leader. State: Replica: d5d236d712aa4f3f88093b1d06726d10, State: Running, Role: LEADER
07:01:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.867247 29353 consensus_queue.cc:237] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d5d236d712aa4f3f88093b1d06726d10" member_type: VOTER last_known_addr { host: "127.27.232.193" port: 33459 } }
07:01:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.870752 29347 ts_tablet_manager.cc:1398] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10: Time spent starting tablet: real 0.055s	user 0.034s	sys 0.010s
07:01:07.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.886788 28778 catalog_manager.cc:5484] T 8cf7c867c20f4638b9590c022978414e P d5d236d712aa4f3f88093b1d06726d10 reported cstate change: term changed from 0 to 1, leader changed from <none> to d5d236d712aa4f3f88093b1d06726d10 (127.27.232.193). New cstate: current_term: 1 leader_uuid: "d5d236d712aa4f3f88093b1d06726d10" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d5d236d712aa4f3f88093b1d06726d10" member_type: VOTER last_known_addr { host: "127.27.232.193" port: 33459 } health_report { overall_health: HEALTHY } } }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.951205 28778 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53482:
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "val"
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
07:01:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:07.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.968339 29297 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:a0c99eb2ea1b4642920d048cd5326cf0)
07:01:07.976 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 51448bca14414aedacbf131cc14730fb
07:01:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.996789 29084 tablet_service.cc:1467] Processing CreateTablet for tablet 2188f5851fe843589132a3a9d8390e17 (DEFAULT_TABLE table=simple-test [id=51448bca14414aedacbf131cc14730fb]), partition=RANGE (key) PARTITION UNBOUNDED
07:01:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.998237 29084 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2188f5851fe843589132a3a9d8390e17. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:08.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.020648 29360 tablet_bootstrap.cc:492] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: Bootstrap starting.
07:01:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.025686 29360 tablet_bootstrap.cc:654] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: Neither blocks nor log segments found. Creating new log.
07:01:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.027251 29360 log.cc:824] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: Log is configured to *not* fsync() on all Append() calls
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.031358 29360 tablet_bootstrap.cc:492] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: No bootstrap required, opened a new log
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.031744 29360 ts_tablet_manager.cc:1367] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: Time spent bootstrapping tablet: real 0.011s	user 0.009s	sys 0.000s
07:01:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.051908 29360 raft_consensus.cc:348] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "873d5cd21db94f939e82ce1c854ff24c" member_type: VOTER last_known_addr { host: "127.27.232.194" port: 41829 } }
07:01:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.052525 29360 raft_consensus.cc:374] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.052778 29360 raft_consensus.cc:729] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 873d5cd21db94f939e82ce1c854ff24c, State: Initialized, Role: FOLLOWER
07:01:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.053409 29360 consensus_queue.cc:260] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "873d5cd21db94f939e82ce1c854ff24c" member_type: VOTER last_known_addr { host: "127.27.232.194" port: 41829 } }
07:01:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.053910 29360 raft_consensus.cc:388] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:01:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.054216 29360 raft_consensus.cc:482] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:01:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.054576 29360 raft_consensus.cc:3037] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 0 FOLLOWER]: Advancing to term 1
07:01:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.057171 29015 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:8cf7c867c20f4638b9590c022978414e)
07:01:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.058496 29360 raft_consensus.cc:504] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "873d5cd21db94f939e82ce1c854ff24c" member_type: VOTER last_known_addr { host: "127.27.232.194" port: 41829 } }
07:01:08.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.059180 29360 leader_election.cc:302] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 873d5cd21db94f939e82ce1c854ff24c; no voters: 
07:01:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.060627 29360 leader_election.cc:288] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [CANDIDATE]: Term 1 election: Requested vote from peers 
07:01:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.061306 29362 raft_consensus.cc:2781] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 1 FOLLOWER]: Leader election won for term 1
07:01:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.063448 29362 raft_consensus.cc:686] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [term 1 LEADER]: Becoming Leader. State: Replica: 873d5cd21db94f939e82ce1c854ff24c, State: Running, Role: LEADER
07:01:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.064057 29360 ts_tablet_manager.cc:1398] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c: Time spent starting tablet: real 0.032s	user 0.032s	sys 0.000s
07:01:08.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.064405 29362 consensus_queue.cc:237] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "873d5cd21db94f939e82ce1c854ff24c" member_type: VOTER last_known_addr { host: "127.27.232.194" port: 41829 } }
07:01:08.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.077044 28778 catalog_manager.cc:5484] T 2188f5851fe843589132a3a9d8390e17 P 873d5cd21db94f939e82ce1c854ff24c reported cstate change: term changed from 0 to 1, leader changed from <none> to 873d5cd21db94f939e82ce1c854ff24c (127.27.232.194). New cstate: current_term: 1 leader_uuid: "873d5cd21db94f939e82ce1c854ff24c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "873d5cd21db94f939e82ce1c854ff24c" member_type: VOTER last_known_addr { host: "127.27.232.194" port: 41829 } health_report { overall_health: HEALTHY } } }
07:01:08.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:08.113 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:08.113 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:08.113 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [3ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=null) after lookup
07:01:08.115 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:08.116 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] Successfully connected to peer
07:01:08.127 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:01:08.130 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb86c8741, L:/127.0.0.1:45424 - R:/127.27.232.195:38145] using TOKEN/null
07:01:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.201843 29157 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:2188f5851fe843589132a3a9d8390e17)
07:01:08.210 [INFO - main] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-taskdgzdJQ/java/build/dist-test/spark-warehouse').
07:01:08.211 [INFO - main] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-taskdgzdJQ/java/build/dist-test/spark-warehouse'.
07:01:08.219 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 498c2ea76ae249288b7ddbc5567665e8
07:01:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.722074 29297 maintenance_manager.cc:419] P 8b5341cc970c4a9b8909d7fce32b878f: Scheduling FlushMRSOp(a0c99eb2ea1b4642920d048cd5326cf0): perf score=1.000000
07:01:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.988032 29182 maintenance_manager.cc:643] P 8b5341cc970c4a9b8909d7fce32b878f: FlushMRSOp(a0c99eb2ea1b4642920d048cd5326cf0) complete. Timing: real 0.265s	user 0.191s	sys 0.042s Metrics: {"bytes_written":39637,"cfile_init":1,"compiler_manager_pool.queue_time_us":1608,"dirs.queue_time_us":29292,"dirs.run_cpu_time_us":309,"dirs.run_wall_time_us":9007,"drs_written":1,"lbm_read_time_us":224,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6604,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":29371,"threads_started":2}
07:01:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.997074 29297 maintenance_manager.cc:419] P 8b5341cc970c4a9b8909d7fce32b878f: Scheduling UndoDeltaBlockGCOp(a0c99eb2ea1b4642920d048cd5326cf0): 1356 bytes on disk
07:01:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.000304 29182 maintenance_manager.cc:643] P 8b5341cc970c4a9b8909d7fce32b878f: UndoDeltaBlockGCOp(a0c99eb2ea1b4642920d048cd5326cf0) complete. Timing: real 0.001s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":249,"lbm_reads_lt_1ms":4}
07:01:09.051 [DEBUG - main] (Connection.java:748) [peer d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)] connecting to peer
07:01:09.054 [DEBUG - kudu-nio-0] (Connection.java:757) [peer d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)] Successfully connected to peer
07:01:09.066 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:01:09.069 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xac011f69, L:/127.0.0.1:46392 - R:/127.27.232.193:33459] using TOKEN/null
07:01:09.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.311378 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=0.969121
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.578102 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.264s	user 0.124s	sys 0.046s Metrics: {"bytes_written":39350,"cfile_init":1,"compiler_manager_pool.queue_time_us":2114,"dirs.queue_time_us":7410,"dirs.run_cpu_time_us":316,"dirs.run_wall_time_us":6217,"drs_written":1,"lbm_read_time_us":309,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6943,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":17,"thread_start_us":8960,"threads_started":2}
07:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.585649 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 330 bytes on disk
07:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.587333 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":217,"lbm_reads_lt_1ms":4}
07:01:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.839490 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.004873 29297 maintenance_manager.cc:419] P 8b5341cc970c4a9b8909d7fce32b878f: Scheduling FlushMRSOp(a0c99eb2ea1b4642920d048cd5326cf0): perf score=0.969062
07:01:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.048677 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.208s	user 0.149s	sys 0.010s Metrics: {"bytes_written":39528,"cfile_init":1,"dirs.queue_time_us":75,"dirs.run_cpu_time_us":252,"dirs.run_wall_time_us":1557,"drs_written":1,"lbm_read_time_us":257,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6661,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28}
07:01:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.050756 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 446 bytes on disk
07:01:10.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.052819 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":233,"lbm_reads_lt_1ms":4}
07:01:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.234272 29182 maintenance_manager.cc:643] P 8b5341cc970c4a9b8909d7fce32b878f: FlushMRSOp(a0c99eb2ea1b4642920d048cd5326cf0) complete. Timing: real 0.229s	user 0.127s	sys 0.040s Metrics: {"bytes_written":39324,"cfile_init":1,"dirs.queue_time_us":7347,"dirs.run_cpu_time_us":278,"dirs.run_wall_time_us":9611,"drs_written":1,"lbm_read_time_us":328,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6083,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":17,"thread_start_us":58956,"threads_started":2,"wal-append.queue_time_us":51700}
07:01:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.237335 29297 maintenance_manager.cc:419] P 8b5341cc970c4a9b8909d7fce32b878f: Scheduling UndoDeltaBlockGCOp(a0c99eb2ea1b4642920d048cd5326cf0): 330 bytes on disk
07:01:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.238958 29182 maintenance_manager.cc:643] P 8b5341cc970c4a9b8909d7fce32b878f: UndoDeltaBlockGCOp(a0c99eb2ea1b4642920d048cd5326cf0) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":220,"lbm_reads_lt_1ms":4}
07:01:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.564666 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:10.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.863811 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.298s	user 0.197s	sys 0.027s Metrics: {"bytes_written":39910,"cfile_init":1,"dirs.queue_time_us":26795,"dirs.run_cpu_time_us":315,"dirs.run_wall_time_us":1712,"drs_written":1,"lbm_read_time_us":286,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11402,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":43,"thread_start_us":26463,"threads_started":1}
07:01:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.866096 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 617 bytes on disk
07:01:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.867810 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":251,"lbm_reads_lt_1ms":4}
07:01:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.121119 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:11.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.295147 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.173s	user 0.137s	sys 0.022s Metrics: {"bytes_written":39637,"cfile_init":1,"dirs.queue_time_us":156,"dirs.run_cpu_time_us":239,"dirs.run_wall_time_us":1439,"drs_written":1,"lbm_read_time_us":217,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6921,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":29}
07:01:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.297542 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 461 bytes on disk
07:01:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.312251 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":282,"lbm_reads_lt_1ms":4}
07:01:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.815796 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.089116 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.273s	user 0.197s	sys 0.014s Metrics: {"bytes_written":39894,"cfile_init":1,"dirs.queue_time_us":9638,"dirs.run_cpu_time_us":292,"dirs.run_wall_time_us":7310,"drs_written":1,"lbm_read_time_us":290,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6722,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43,"thread_start_us":9843,"threads_started":1}
07:01:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.091722 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 616 bytes on disk
07:01:12.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.093377 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":204,"lbm_reads_lt_1ms":4}
07:01:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.345446 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.565043 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.219s	user 0.182s	sys 0.004s Metrics: {"bytes_written":39759,"cfile_init":1,"dirs.queue_time_us":84,"dirs.run_cpu_time_us":248,"dirs.run_wall_time_us":1583,"drs_written":1,"lbm_read_time_us":310,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6948,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34}
07:01:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.567371 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 517 bytes on disk
07:01:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.569393 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":243,"lbm_reads_lt_1ms":4}
07:01:12.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.823925 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.968659 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.144s	user 0.121s	sys 0.020s Metrics: {"bytes_written":39786,"cfile_init":1,"dirs.queue_time_us":86,"dirs.run_cpu_time_us":289,"dirs.run_wall_time_us":1629,"drs_written":1,"lbm_read_time_us":204,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5053,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:01:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.972949 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 571 bytes on disk
07:01:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.974511 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":158,"lbm_reads_lt_1ms":4}
07:01:13.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.227001 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.373450 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.146s	user 0.135s	sys 0.007s Metrics: {"bytes_written":39851,"cfile_init":1,"dirs.queue_time_us":100,"dirs.run_cpu_time_us":305,"dirs.run_wall_time_us":1545,"drs_written":1,"lbm_read_time_us":197,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5671,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:01:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.375363 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 571 bytes on disk
07:01:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.376721 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":156,"lbm_reads_lt_1ms":4}
07:01:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.629375 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.807274 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.177s	user 0.151s	sys 0.006s Metrics: {"bytes_written":39926,"cfile_init":1,"dirs.queue_time_us":92,"dirs.run_cpu_time_us":304,"dirs.run_wall_time_us":1781,"drs_written":1,"lbm_read_time_us":199,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7849,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":41}
07:01:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.809041 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 597 bytes on disk
07:01:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.810827 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":231,"lbm_reads_lt_1ms":4}
07:01:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.063481 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.222877 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.159s	user 0.144s	sys 0.008s Metrics: {"bytes_written":39778,"cfile_init":1,"dirs.queue_time_us":98,"dirs.run_cpu_time_us":246,"dirs.run_wall_time_us":1727,"drs_written":1,"lbm_read_time_us":225,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5212,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:01:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.227499 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 581 bytes on disk
07:01:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.229360 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":220,"lbm_reads_lt_1ms":4}
07:01:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.482841 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.633690 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.150s	user 0.141s	sys 0.008s Metrics: {"bytes_written":39779,"cfile_init":1,"dirs.queue_time_us":79,"dirs.run_cpu_time_us":201,"dirs.run_wall_time_us":1455,"drs_written":1,"lbm_read_time_us":158,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5099,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:01:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.636166 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 582 bytes on disk
07:01:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.641010 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":220,"lbm_reads_lt_1ms":4}
07:01:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.893294 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.051985 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.158s	user 0.144s	sys 0.012s Metrics: {"bytes_written":39882,"cfile_init":1,"dirs.queue_time_us":80,"dirs.run_cpu_time_us":240,"dirs.run_wall_time_us":1716,"drs_written":1,"lbm_read_time_us":142,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6029,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41,"spinlock_wait_cycles":7040}
07:01:15.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.053958 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 596 bytes on disk
07:01:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.055476 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":178,"lbm_reads_lt_1ms":4}
07:01:15.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.308344 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.453295 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.144s	user 0.135s	sys 0.008s Metrics: {"bytes_written":39867,"cfile_init":1,"dirs.queue_time_us":79,"dirs.run_cpu_time_us":277,"dirs.run_wall_time_us":1431,"drs_written":1,"lbm_read_time_us":153,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4812,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:01:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.458536 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 604 bytes on disk
07:01:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.460327 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":201,"lbm_reads_lt_1ms":4}
07:01:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.713105 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:15.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.866865 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.153s	user 0.144s	sys 0.004s Metrics: {"bytes_written":39895,"cfile_init":1,"dirs.queue_time_us":96,"dirs.run_cpu_time_us":276,"dirs.run_wall_time_us":1619,"drs_written":1,"lbm_read_time_us":210,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5122,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:01:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.868919 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 607 bytes on disk
07:01:15.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.870626 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":158,"lbm_reads_lt_1ms":4}
07:01:16.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.123629 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.253016 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.129s	user 0.123s	sys 0.004s Metrics: {"bytes_written":39868,"cfile_init":1,"dirs.queue_time_us":96,"dirs.run_cpu_time_us":284,"dirs.run_wall_time_us":1469,"drs_written":1,"lbm_read_time_us":160,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4525,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:01:16.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.255527 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 606 bytes on disk
07:01:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.257517 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":175,"lbm_reads_lt_1ms":4}
07:01:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.510030 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.653359 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.143s	user 0.138s	sys 0.000s Metrics: {"bytes_written":39805,"cfile_init":1,"dirs.queue_time_us":83,"dirs.run_cpu_time_us":258,"dirs.run_wall_time_us":1546,"drs_written":1,"lbm_read_time_us":150,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4859,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:01:16.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.656564 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 562 bytes on disk
07:01:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.659214 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":251,"lbm_reads_lt_1ms":4}
07:01:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.912779 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.067631 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.154s	user 0.146s	sys 0.004s Metrics: {"bytes_written":39677,"cfile_init":1,"dirs.queue_time_us":75,"dirs.run_cpu_time_us":218,"dirs.run_wall_time_us":1483,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5459,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:01:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.069782 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 581 bytes on disk
07:01:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.071911 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.002s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":235,"lbm_reads_lt_1ms":4}
07:01:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.324908 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.472978 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.147s	user 0.141s	sys 0.004s Metrics: {"bytes_written":39934,"cfile_init":1,"dirs.queue_time_us":95,"dirs.run_cpu_time_us":278,"dirs.run_wall_time_us":1825,"drs_written":1,"lbm_read_time_us":200,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4492,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":44}
07:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.475270 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 625 bytes on disk
07:01:17.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.477021 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":150,"lbm_reads_lt_1ms":4}
07:01:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.730569 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.891286 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.160s	user 0.150s	sys 0.004s Metrics: {"bytes_written":39861,"cfile_init":1,"dirs.queue_time_us":83,"dirs.run_cpu_time_us":281,"dirs.run_wall_time_us":1727,"drs_written":1,"lbm_read_time_us":140,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5337,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:01:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.894639 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 572 bytes on disk
07:01:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.897197 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":267,"lbm_reads_lt_1ms":4}
07:01:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.151731 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.306320 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.154s	user 0.147s	sys 0.004s Metrics: {"bytes_written":39938,"cfile_init":1,"dirs.queue_time_us":83,"dirs.run_cpu_time_us":229,"dirs.run_wall_time_us":1268,"drs_written":1,"lbm_read_time_us":147,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4921,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42,"spinlock_wait_cycles":1280}
07:01:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.308894 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 606 bytes on disk
07:01:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.311105 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4}
07:01:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.564953 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.727519 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.162s	user 0.139s	sys 0.012s Metrics: {"bytes_written":40009,"cfile_init":1,"dirs.queue_time_us":107,"dirs.run_cpu_time_us":263,"dirs.run_wall_time_us":1593,"drs_written":1,"lbm_read_time_us":184,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5166,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":44}
07:01:18.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.730033 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 627 bytes on disk
07:01:18.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.732300 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":218,"lbm_reads_lt_1ms":4}
07:01:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:18.903584 28809 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab80 after lost signal to thread 28741
07:01:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:18.904601 28809 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2d60 after lost signal to thread 28812
07:01:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:18.985925 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.156798 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.170s	user 0.161s	sys 0.004s Metrics: {"bytes_written":39964,"cfile_init":1,"dirs.queue_time_us":79,"dirs.run_cpu_time_us":320,"dirs.run_wall_time_us":1753,"drs_written":1,"lbm_read_time_us":170,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5090,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43}
07:01:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.160364 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 616 bytes on disk
07:01:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.162710 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.002s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":160,"lbm_reads_lt_1ms":4}
07:01:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.417106 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.630887 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.213s	user 0.173s	sys 0.012s Metrics: {"bytes_written":39839,"cfile_init":1,"dirs.queue_time_us":70,"dirs.run_cpu_time_us":259,"dirs.run_wall_time_us":1540,"drs_written":1,"lbm_read_time_us":205,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6218,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":35,"spinlock_wait_cycles":1792}
07:01:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.634656 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 526 bytes on disk
07:01:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.637995 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.003s	user 0.002s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":226,"lbm_reads_lt_1ms":4}
07:01:19.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.892508 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=1.000000
07:01:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:20.058341 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.165s	user 0.150s	sys 0.004s Metrics: {"bytes_written":39862,"cfile_init":1,"dirs.queue_time_us":127,"dirs.run_cpu_time_us":288,"dirs.run_wall_time_us":1632,"drs_written":1,"lbm_read_time_us":193,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6305,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39,"spinlock_wait_cycles":12160}
07:01:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:20.062319 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 571 bytes on disk
07:01:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:20.067147 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.003s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":1413,"lbm_reads_1-10_ms":1,"lbm_reads_lt_1ms":3}
07:01:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:21.076157 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling FlushMRSOp(8cf7c867c20f4638b9590c022978414e): perf score=0.969180
07:01:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:21.178747 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: FlushMRSOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.102s	user 0.079s	sys 0.020s Metrics: {"bytes_written":39590,"cfile_init":1,"dirs.queue_time_us":744,"dirs.run_cpu_time_us":247,"dirs.run_wall_time_us":1568,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4260,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":26,"thread_start_us":394,"threads_started":1}
07:01:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:21.181059 29015 maintenance_manager.cc:419] P d5d236d712aa4f3f88093b1d06726d10: Scheduling UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e): 426 bytes on disk
07:01:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:21.182868 28899 maintenance_manager.cc:643] P d5d236d712aa4f3f88093b1d06726d10: UndoDeltaBlockGCOp(8cf7c867c20f4638b9590c022978414e) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":158,"lbm_reads_lt_1ms":4}
07:01:25.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 498c2ea76ae249288b7ddbc5567665e8
07:01:25.322 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for test
07:01:25.322 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2206) Key ranges for test
07:01:25.322 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, 0x80000020), 35457, "a0c99eb2ea1b4642920d048cd5326cf0" [<start>, 0x80000032)
07:01:25.322 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [<start>, 0x80000042), 35170, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000020, <end>), 35144, "a0c99eb2ea1b4642920d048cd5326cf0" [<start>, 0x80000032)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000042, 0x8000005E), 35348, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000005E, 0x80000089), 35730, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000089, 0x800000A6), 35457, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000A6, 0x800000D1), 35714, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000D1, 0x800000F3), 35579, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000F3, 0x8000011A), 35606, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000011A, 0x80000141), 35671, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000141, 0x8000016A), 35746, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000016A, 0x80000192), 35598, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000192, 0x800001BA), 35599, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800001BA, 0x800001E3), 35702, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800001E3, 0x8000020D), 35687, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000020D, 0x80000237), 35715, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000237, 0x80000261), 35688, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000261, 0x80000287), 35625, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000287, 0x800002AF), 35497, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.325 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800002AF, 0x800002DB), 35754, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800002DB, 0x80000302), 35681, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000302, 0x8000032C), 35758, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000032C, 0x80000358), 35829, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000358, 0x80000383), 35784, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000383, 0x800003A6), 35659, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800003A6, 0x800003CD), 35682, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800003CD, <end>), 35410, "8cf7c867c20f4638b9590c022978414e" [0x80000032, <end>)
07:01:25.342 [INFO - main] (Logging.scala:57) Starting job: collectAsList at SparkSQLTest.scala:471
07:01:25.344 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (collectAsList at SparkSQLTest.scala:471) with 27 output partitions
07:01:25.344 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (collectAsList at SparkSQLTest.scala:471)
07:01:25.344 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
07:01:25.344 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
07:01:25.346 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471), which has no missing parents
07:01:25.351 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 25.2 KiB, free 1993.9 MiB)
07:01:25.353 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 9.6 KiB, free 1993.8 MiB)
07:01:25.354 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-9fw8.c.gcp-upstream.internal:39879 (size: 9.6 KiB, free: 1993.9 MiB)
07:01:25.355 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
07:01:25.356 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 27 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))
07:01:25.356 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 27 tasks resource profile 0
07:01:25.358 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-9fw8.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5324 bytes) taskResourceAssignments Map()
07:01:25.360 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
07:01:25.366 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:01:25.421 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 32 rows, more = false, responseScanTimestamp = 7021014570485215232) for scanner KuduScanner(table=test, tablet=a0c99eb2ea1b4642920d048cd5326cf0, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x80000020, lastPrimaryKey=<last>)
07:01:25.467 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 13458 bytes result sent to driver
07:01:25.470 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-9fw8.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5324 bytes) taskResourceAssignments Map()
07:01:25.471 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
07:01:25.471 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 113 ms on dist-test-slave-9fw8.c.gcp-upstream.internal (executor driver) (1/27)
07:01:25.476 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:01:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714114885 (unix time) try "date -d @1714114885" if you are using GNU date ***
07:01:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @                0x0 (unknown)
07:01:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 29160 (TID 0x7efff29ff700) from PID 8; stack trace: ***
07:01:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @           0x44fd80 __tsan::CallUserSignalHandler()
07:01:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @           0x452263 rtl_sigaction()
07:01:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f001238a980 (unknown)
07:01:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018c9252b kudu::Slice::compare()
07:01:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018033c41 kudu::operator<()
07:01:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0011aed7c9 kudu::ScanSpec::SetLowerBoundKey()
07:01:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f001810354f kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
07:01:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018103208 kudu::tablet::CFileSet::Iterator::Init()
07:01:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018226039 kudu::tablet::DeltaApplier::Init()
07:01:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0011a5be73 kudu::MaterializingIterator::Init()
07:01:25.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0011a58b77 kudu::InitAndMaybeWrap()
07:01:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0011a5afb6 kudu::UnionIterator::InitSubIterators()
07:01:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0011a5ab13 kudu::UnionIterator::Init()
07:01:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f001800bb35 kudu::tablet::Tablet::Iterator::Init()
07:01:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018c67c24 kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
07:01:25.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0018c65b60 kudu::tserver::TabletServiceImpl::Scan()
07:01:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f00173042f5 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf()::$_5::operator()()
07:01:25.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0017304283 _ZNSt3__18__invokeIRZN4kudu7tserver21TabletServerServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE3$_5JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
07:01:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f00173041b2 std::__1::__invoke_void_return_wrapper<>::__call<>()
07:01:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f001730412d std::__1::__function::__alloc_func<>::operator()()
07:01:25.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f00173033e3 std::__1::__function::__func<>::operator()()
07:01:25.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a32a4d std::__1::__function::__value_func<>::operator()()
07:01:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a31e87 std::__1::function<>::operator()()
07:01:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a31830 kudu::rpc::GeneratedServiceIf::Handle()
07:01:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a34c24 kudu::rpc::ServicePool::RunThread()
07:01:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a35f42 kudu::rpc::ServicePool::Init()::$_0::operator()()
07:01:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a35efa _ZNSt3__18__invokeIRZN4kudu3rpc11ServicePool4InitEiE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_
07:01:25.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a35e8a std::__1::__invoke_void_return_wrapper<>::__call<>()
07:01:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a35e52 std::__1::__function::__alloc_func<>::operator()()
07:01:25.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f0015a3514e std::__1::__function::__func<>::operator()()
07:01:25.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @           0x589485 std::__1::__function::__value_func<>::operator()()
07:01:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @           0x5892ba std::__1::function<>::operator()()
07:01:26.134 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] handling channelInactive
07:01:26.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state READY due to: connection closed
07:01:26.134 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: connection closed
07:01:26.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:26.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:26.150 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:26.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=2, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=2, TimeoutTracker(timeout=30000, elapsed=673), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1)))) after lookup
07:01:26.151 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:26.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.153 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.153 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:26.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:26.170 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:26.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=4, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=4, TimeoutTracker(timeout=30000, elapsed=694), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996)))) after lookup
07:01:26.172 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:26.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.174 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:26.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:26.190 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:26.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=6, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=6, TimeoutTracker(timeout=30000, elapsed=713), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982)))) after lookup
07:01:26.191 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:26.193 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.193 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.209 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:26.209 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:26.210 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:26.210 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=8, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=8, TimeoutTracker(timeout=30000, elapsed=733), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249)))) after lookup
07:01:26.212 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:26.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.213 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.349 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:26.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:26.350 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:26.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=10, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=10, TimeoutTracker(timeout=30000, elapsed=873), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196)))) after lookup
07:01:26.351 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:26.353 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:26.353 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:27.370 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:27.370 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:27.371 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:27.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=12, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=12, TimeoutTracker(timeout=30000, elapsed=1894), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259)))) after lookup
07:01:27.372 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:27.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:27.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:27.374 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.470 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:31.470 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:31.470 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:31.471 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=14, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=14, TimeoutTracker(timeout=30000, elapsed=5994), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442)))) after lookup
07:01:31.472 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:31.473 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.474 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:31.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:31.990 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:31.991 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=16, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=16, TimeoutTracker(timeout=30000, elapsed=6514), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979)))) after lookup
07:01:31.992 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:31.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:31.993 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:33.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:33.751 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:33.751 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:33.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=18, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=18, TimeoutTracker(timeout=30000, elapsed=8275), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209)))) after lookup
07:01:33.753 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:33.755 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:33.755 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:33.756 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.050 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:36.050 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:36.051 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:36.051 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=20, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=20, TimeoutTracker(timeout=30000, elapsed=10574), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393)))) after lookup
07:01:36.052 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:36.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.053 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:36.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:36.150 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:36.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=22, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=22, TimeoutTracker(timeout=30000, elapsed=10674), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810)))) after lookup
07:01:36.152 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:36.154 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.154 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:36.155 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:37.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:37.069 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:37.069 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:37.070 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=24, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=24, TimeoutTracker(timeout=30000, elapsed=11593), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549)))) after lookup
07:01:37.072 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:37.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:37.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:37.073 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:38.010 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:38.010 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:38.011 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:38.011 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=26, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=26, TimeoutTracker(timeout=30000, elapsed=12534), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953)))) after lookup
07:01:38.012 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:38.014 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:38.014 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:38.015 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:40.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:40.291071 29146 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7520 after lost signal to thread 29019
07:01:40.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:40.291972 29146 debug-util.cc:398] Leaking SignalData structure 0x7b08000d4be0 after lost signal to thread 29149
07:01:41.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:41.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:41.290 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:41.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=28, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=28, TimeoutTracker(timeout=30000, elapsed=15814), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373)))) after lookup
07:01:41.292 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:41.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:41.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:41.293 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:43.349 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:43.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:43.350 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:43.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=30, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=30, TimeoutTracker(timeout=30000, elapsed=17874), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270)))) after lookup
07:01:43.352 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:43.353 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:43.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:43.354 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:45.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:45.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:45.270 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:45.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=32, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=32, TimeoutTracker(timeout=30000, elapsed=19794), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012)))) after lookup
07:01:45.272 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:45.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:45.274 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:45.274 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:46.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:46.510676 29146 debug-util.cc:398] Leaking SignalData structure 0x7b08000d56c0 after lost signal to thread 29019
07:01:46.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:46.511489 29146 debug-util.cc:398] Leaking SignalData structure 0x7b08000d5200 after lost signal to thread 29149
07:01:47.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:47.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:47.491 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:47.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=34, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=34, TimeoutTracker(timeout=30000, elapsed=22015), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258)))) after lookup
07:01:47.494 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:47.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:47.496 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:47.496 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:49.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:49.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:49.771 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:49.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=36, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=36, TimeoutTracker(timeout=30000, elapsed=24295), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313)))) after lookup
07:01:49.774 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:49.774 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:49.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:49.775 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:51.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:51.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:51.690 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:51.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=38, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=38, TimeoutTracker(timeout=30000, elapsed=26214), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063)))) after lookup
07:01:51.693 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:51.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:51.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:51.695 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:52.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:52.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:52.691 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:52.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=40, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=40, TimeoutTracker(timeout=30000, elapsed=27215), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920)))) after lookup
07:01:52.693 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:52.694 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:52.694 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:52.694 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.770 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:53.770 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:53.770 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:53.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=42, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=42, TimeoutTracker(timeout=30000, elapsed=28295), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479)))) after lookup
07:01:53.774 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:53.776 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.776 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.776 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a0c99eb2ea1b4642920d048cd5326cf0 for table 'test' with partition [<start>, 0x80000032)
07:01:53.870 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8cf7c867c20f4638b9590c022978414e for table 'test' with partition [0x80000032, <end>)
07:01:53.870 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=a0c99eb2ea1b4642920d048cd5326cf0@[8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=8cf7c867c20f4638b9590c022978414e@[d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)[L]]}]
07:01:53.870 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=44, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=44, TimeoutTracker(timeout=30000, elapsed=28393), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216)))) after lookup
07:01:53.872 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] connecting to peer
07:01:53.873 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.873 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145) for tablet a0c99eb2ea1b4642920d048cd5326cf0: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
07:01:53.874 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
07:01:53.874 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:01:53.889 [ERROR - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:94) Exception in task 1.0 in stage 0.0 (TID 1)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160) ~[main/:?]
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:01:53.957 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-9fw8.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5324 bytes) taskResourceAssignments Map()
07:01:53.958 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
07:01:53.968 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:01:53.968 [WARN - task-result-getter-1] (Logging.scala:69) Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-9fw8.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:01:53.972 [ERROR - task-result-getter-1] (Logging.scala:73) Task 1 in stage 0.0 failed 1 times; aborting job
07:01:53.976 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
07:01:53.976 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
07:01:53.982 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Executor is trying to kill task 2.0 in stage 0.0 (TID 2), reason: Stage cancelled
07:01:53.983 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
07:01:53.988 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (collectAsList at SparkSQLTest.scala:471) failed in 28.635 s due to Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-9fw8.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
07:01:53.991 [INFO - main] (Logging.scala:57) Job 0 failed: collectAsList at SparkSQLTest.scala:471, took 28.647670 s
07:01:54.001 [INFO - dispatcher-event-loop-1] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
07:01:54.006 [INFO - main] (Logging.scala:57) MemoryStore cleared
07:01:54.006 [INFO - main] (Logging.scala:57) BlockManager stopped
07:01:54.007 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
07:01:54.007 [INFO - dispatcher-event-loop-1] (Logging.scala:57) OutputCommitCoordinator stopped!
07:01:54.014 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
07:01:54.014 [DEBUG - main] (Connection.java:575) [peer master-127.27.232.252:45485(127.27.232.252:45485)] disconnecting while in state READY
07:01:54.014 [DEBUG - main] (Connection.java:575) [peer d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)] disconnecting while in state READY
07:01:54.016 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.232.252:45485(127.27.232.252:45485)] handling channelInactive
07:01:54.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.232.252:45485(127.27.232.252:45485)] cleaning up while in state READY due to: connection closed
07:01:54.017 [DEBUG - kudu-nio-0] (Connection.java:239) [peer d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)] handling channelInactive
07:01:54.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459)] cleaning up while in state READY due to: connection closed
07:01:54.017 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d5d236d712aa4f3f88093b1d06726d10(127.27.232.193:33459) for tablet 8cf7c867c20f4638b9590c022978414e: connection closed
07:01:54.017 [DEBUG - main] (Connection.java:575) [peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145)] disconnecting while in state TERMINATED
07:01:54.018 [DEBUG - main] (Connection.java:575) [peer master-127.27.232.254:42503(127.27.232.254:42503)] disconnecting while in state READY
07:01:54.018 [DEBUG - main] (Connection.java:575) [peer master-127.27.232.253:33513(127.27.232.253:33513)] disconnecting while in state READY
07:01:54.019 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.232.254:42503(127.27.232.254:42503)] handling channelInactive
07:01:54.020 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.232.254:42503(127.27.232.254:42503)] cleaning up while in state READY due to: connection closed
07:01:54.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:01:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:54.020798 28897 connection.cc:441] server connection from 127.0.0.1:46392 torn down before Call kudu.tserver.TabletServerService.Scan from 127.0.0.1:46392 (request call id 951) could send its response
07:01:54.021 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.232.253:33513(127.27.232.253:33513)] handling channelInactive
07:01:54.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.232.253:33513(127.27.232.253:33513)] cleaning up while in state READY due to: connection closed
07:01:54.021 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 36 thread-local buffer(s) from thread: kudu-nio-1
07:01:54.022 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 30 thread-local buffer(s) from thread: kudu-nio-0
07:01:54.033 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:01:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:54.035120 28579 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu with pid 28860
07:01:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:54.072911 28579 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu with pid 29016
07:01:54.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:54.106812 28579 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu with pid 28591
07:01:54.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:54.137578 28579 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu with pid 28661
07:01:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:54.172556 28579 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskdgzdJQ/build/tsan/bin/kudu with pid 28736
07:01:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:01:55Z chronyd exiting
07:01:56.300 [ERROR - main] (RetryRule.java:206) org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange: failed attempt 1
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-9fw8.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-9fw8.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, KuduRpc(method=Scan, tablet=a0c99eb2ea1b4642920d048cd5326cf0, attempt=45, TimeoutTracker(timeout=30000, elapsed=28396), Traces: [0ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [657ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: connection closed, [657ms] delaying RPC due to: Network error: connection closed, [669ms] refreshing cache from master, [669ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [674ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [676ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [677ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [690ms] refreshing cache from master, [690ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [692ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [695ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [697ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [697ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [710ms] refreshing cache from master, [710ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [712ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [714ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [716ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [716ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [729ms] refreshing cache from master, [729ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [732ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [734ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [736ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [736ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [870ms] refreshing cache from master, [870ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [874ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [876ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1890ms] refreshing cache from master, [1890ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [1893ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [1895ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [1897ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [1897ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5990ms] refreshing cache from master, [5990ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [5992ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [5994ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [5996ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [5997ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6510ms] refreshing cache from master, [6510ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [6512ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [6515ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [6516ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [6516ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8270ms] refreshing cache from master, [8270ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [8273ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [8276ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [8279ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [8279ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10570ms] refreshing cache from master, [10570ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10573ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10575ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10576ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10577ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10670ms] refreshing cache from master, [10670ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [10672ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [10675ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [10677ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [10678ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11589ms] refreshing cache from master, [11589ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [11592ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [11594ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [11596ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [11596ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12530ms] refreshing cache from master, [12530ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [12533ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [12535ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [12538ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [12538ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15809ms] refreshing cache from master, [15809ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [15812ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [15815ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [15816ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [15816ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17869ms] refreshing cache from master, [17869ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [17872ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [17875ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [17876ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [17877ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19789ms] refreshing cache from master, [19789ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.232.252:45485, [19792ms] Sub RPC GetTableLocations: received response from server master-127.27.232.252:45485: OK, [19795ms] sending RPC to server 8b5341cc970c4a9b8909d7fce32b878f, [19797ms] received response from server 8b5341cc970c4a9b8909d7fce32b878f: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [19797ms] delaying RPC due to: Network error: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145, [22009ms] refreshing cache from master, [22010ms] too many traces: truncated at 100 traces, deferred=Deferred@177232285(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1941801303 after retry RPC@2070505996) -> (continuation of Deferred@516316459 after retry RPC@1384725982) -> (continuation of Deferred@129624677 after retry RPC@482693249) -> (continuation of Deferred@1467980029 after retry RPC@1402950196) -> (continuation of Deferred@1985111783 after retry RPC@1601537259) -> (continuation of Deferred@823896662 after retry RPC@960955442) -> (continuation of Deferred@715991248 after retry RPC@726322979) -> (continuation of Deferred@296206814 after retry RPC@1137789209) -> (continuation of Deferred@1723056207 after retry RPC@795067393) -> (continuation of Deferred@240580996 after retry RPC@703301810) -> (continuation of Deferred@902307734 after retry RPC@1083420549) -> (continuation of Deferred@490490253 after retry RPC@1675086953) -> (continuation of Deferred@1387725638 after retry RPC@257829373) -> (continuation of Deferred@1089703308 after retry RPC@1330685270) -> (continuation of Deferred@332314246 after retry RPC@1501486012) -> (continuation of Deferred@2036129127 after retry RPC@1713064258) -> (continuation of Deferred@847034397 after retry RPC@244426313) -> (continuation of Deferred@2101426604 after retry RPC@1177859063) -> (continuation of Deferred@871682610 after retry RPC@1354527920) -> (continuation of Deferred@254456528 after retry RPC@219414479) -> (continuation of Deferred@861855131 after retry RPC@555852216) -> (continuation of Deferred@67216723 after retry RPC@2075460912))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 8b5341cc970c4a9b8909d7fce32b878f(127.27.232.195:38145): Connection refused: /127.27.232.195:38145
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.232.195:38145
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)
	at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
	at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
	at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696)
	at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977)
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)
	at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976)
	at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
	at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)