Unable to diagnose

06:59:14.564 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:59:14.565 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ9V9WF/test-tmp
06:59:14.565 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
06:59:14.565 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ9V9WF/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
06:59:14.571 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423"
  extra_tserver_flags: "--flush_threshold_mb=1"
  extra_tserver_flags: "--flush_threshold_secs=1"
  extra_tserver_flags: "--enable_rowset_compaction=false"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

06:59:14.876 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:59:14.876 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

06:59:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.878659 19623 mini_oidc.cc:152] Starting JWKS server
06:59:14.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.881042 19623 webserver.cc:460] Webserver started at http://127.0.0.1:36855/ using document root <none> and password file <none>
06:59:14.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.881247 19623 mini_oidc.cc:189] Starting OIDC Discovery server
06:59:14.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.881767 19623 webserver.cc:460] Webserver started at http://127.0.0.1:34985/ using document root <none> and password file <none>
06:59:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:14Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:59:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:14Z Disabled control of system clock
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.911209 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/logs
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data/info.pb
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.254:38787
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.254
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json with env {}
06:59:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.221179 19634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.221486 19634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.221587 19634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229051 19634 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229161 19634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229207 19634 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229256 19634 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229295 19634 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.229341 19634 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36855/jwks.json
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.238437 19634 master_runner.cc:386] Master server non-default flags:
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data
06:59:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.254:38787
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data/info.pb
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.254
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/logs
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.239782 19634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.241523 19634 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.252837 19639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.253415 19640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.254343 19642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.254932 19634 server_base.cc:1034] running on GCE node
06:59:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.255692 19634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.257094 19634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.258316 19634 hybrid_clock.cc:648] HybridClock initialized: now 1714114755258248 us; error 42 us; skew 500 ppm
06:59:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.258673 19634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.265555 19634 webserver.cc:460] Webserver started at http://127.19.41.254:37683/ using document root <none> and password file <none>
06:59:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.266256 19634 fs_manager.cc:359] Metadata directory not provided
06:59:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.266381 19634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.266731 19634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:15.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.269280 19634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data/instance:
06:59:15.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6"
06:59:15.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.270035 19634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal/instance:
06:59:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6"
06:59:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.275045 19634 fs_manager.cc:691] Time spent creating directory manager: real 0.005s	user 0.008s	sys 0.000s
06:59:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.278187 19648 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.279390 19634 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.279552 19634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6"
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.279738 19634 fs_report.cc:383] FS layout report
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data/data
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:15.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.306056 19634 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:15.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.306900 19634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:15.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.307235 19634 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.324143 19634 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.254:38787
06:59:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.324189 19699 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.254:38787 every 8 connection(s)
06:59:15.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.325398 19634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/data/info.pb
06:59:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.328820 19700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:15.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.334338 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } has no permanent_uuid. Determining permanent_uuid...
06:59:15.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.336019 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 19634
06:59:15.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.336346 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-0/wal/instance
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.339135 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/logs
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data/info.pb
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.253:43055
06:59:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.253
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json with env {}
06:59:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.349098 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
06:59:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.350067 19650 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.351539 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.253:43055: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.400928 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } attempt: 1
06:59:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.402760 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.253:43055: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.470993 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } attempt: 2
06:59:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.472875 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.253:43055: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.564095 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } attempt: 3
06:59:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.565873 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.253:43055: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.649703 19704 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.649977 19704 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.650108 19704 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657393 19704 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657526 19704 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657590 19704 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657665 19704 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657725 19704 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.657799 19704 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36855/jwks.json
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.666446 19704 master_runner.cc:386] Master server non-default flags:
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.253:43055
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data/info.pb
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.253
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/logs
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:15.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.667994 19704 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:15.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.669625 19704 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.679852 19712 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.680673 19713 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.681452 19715 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.682332 19704 server_base.cc:1034] running on GCE node
06:59:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.683066 19704 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:15.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.684376 19704 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.685578 19704 hybrid_clock.cc:648] HybridClock initialized: now 1714114755685521 us; error 32 us; skew 500 ppm
06:59:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.685858 19704 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.687979 19704 webserver.cc:460] Webserver started at http://127.19.41.253:40225/ using document root <none> and password file <none>
06:59:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.688627 19704 fs_manager.cc:359] Metadata directory not provided
06:59:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.688742 19704 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.689026 19704 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.691416 19704 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data/instance:
06:59:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5671858a5f3047cfbe16b39141fdcf33"
06:59:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.692138 19704 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal/instance:
06:59:15.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5671858a5f3047cfbe16b39141fdcf33"
06:59:15.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.696640 19704 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.005s	sys 0.000s
06:59:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.699708 19721 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.700799 19704 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.700973 19704 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5671858a5f3047cfbe16b39141fdcf33"
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:15 on dist-test-slave-7lpv"
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.701148 19704 fs_report.cc:383] FS layout report
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data/data
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:15.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.709069 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } attempt: 4
06:59:15.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.710608 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.253:43055: Network error: Client connection negotiation failed: client connection to 127.19.41.253:43055: connect: Connection refused (error 111)
06:59:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.732461 19704 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.733222 19704 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.733547 19704 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.750694 19704 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.253:43055
06:59:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.750718 19772 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.253:43055 every 8 connection(s)
06:59:15.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.751957 19704 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/data/info.pb
06:59:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.755429 19773 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.756708 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 19704
06:59:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.756893 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-1/wal/instance
06:59:15.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.757848 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/logs
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data/info.pb
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.252:38057
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.252
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:15.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json with env {}
06:59:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.760813 19773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } has no permanent_uuid. Determining permanent_uuid...
06:59:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.772809 19773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
06:59:15.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.780925 19773 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } has no permanent_uuid. Determining permanent_uuid...
06:59:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.781807 19723 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.783311 19773 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.832670 19773 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 1
06:59:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.834630 19773 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.902835 19773 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 2
06:59:15.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.904856 19773 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.996157 19773 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 3
06:59:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:15.998127 19773 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.009831 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } attempt: 5
06:59:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.015846 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } has no permanent_uuid. Determining permanent_uuid...
06:59:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.017393 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.068675 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 1
06:59:16.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.070484 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.073397 19775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.073661 19775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.073760 19775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081091 19775 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081182 19775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081225 19775 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081274 19775 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081313 19775 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.081359 19775 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36855/jwks.json
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.090349 19775 master_runner.cc:386] Master server non-default flags:
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.252:38057
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36855/jwks.json
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data/info.pb
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.252
06:59:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/logs
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.091642 19775 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:16.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.093333 19775 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.103758 19784 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.104369 19786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.105347 19783 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.105984 19775 server_base.cc:1034] running on GCE node
06:59:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.106719 19775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.107993 19775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.109205 19775 hybrid_clock.cc:648] HybridClock initialized: now 1714114756109150 us; error 29 us; skew 500 ppm
06:59:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.109524 19775 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:16.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.111668 19775 webserver.cc:460] Webserver started at http://127.19.41.252:35431/ using document root <none> and password file <none>
06:59:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.112339 19775 fs_manager.cc:359] Metadata directory not provided
06:59:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.112466 19775 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.112754 19775 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.115290 19775 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data/instance:
06:59:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300b834a891d4ae9b17585f0a7481813"
06:59:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:16 on dist-test-slave-7lpv"
06:59:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.116063 19775 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal/instance:
06:59:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300b834a891d4ae9b17585f0a7481813"
06:59:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:16 on dist-test-slave-7lpv"
06:59:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.120836 19775 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.005s	sys 0.000s
06:59:16.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.123947 19792 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.125133 19775 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.125293 19775 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "300b834a891d4ae9b17585f0a7481813"
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:16 on dist-test-slave-7lpv"
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.125469 19775 fs_report.cc:383] FS layout report
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data/data
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:16.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.138669 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 2
06:59:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.140379 19700 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.141371 19775 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.141341 19773 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 4
06:59:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.142244 19775 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.142545 19775 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:16.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.143141 19773 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.41.252:38057: Network error: Client connection negotiation failed: client connection to 127.19.41.252:38057: connect: Connection refused (error 111)
06:59:16.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.159241 19775 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.252:38057
06:59:16.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.159263 19843 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.252:38057 every 8 connection(s)
06:59:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.160533 19775 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/data/info.pb
06:59:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.163601 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 19775
06:59:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.163786 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/master-2/wal/instance
06:59:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.164345 19844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.171206 19844 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } has no permanent_uuid. Determining permanent_uuid...
06:59:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.184000 19844 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
06:59:16.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.190681 19844 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } has no permanent_uuid. Determining permanent_uuid...
06:59:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.199505 19844 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: Bootstrap starting.
06:59:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.202381 19844 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: Neither blocks nor log segments found. Creating new log.
06:59:16.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.203603 19844 log.cc:824] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: Log is configured to *not* fsync() on all Append() calls
06:59:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.205832 19844 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: No bootstrap required, opened a new log
06:59:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.212031 19844 raft_consensus.cc:348] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.212301 19844 raft_consensus.cc:374] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:16.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.212381 19844 raft_consensus.cc:729] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 300b834a891d4ae9b17585f0a7481813, State: Initialized, Role: FOLLOWER
06:59:16.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.212847 19844 consensus_queue.cc:260] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.213603 19851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.213858 19851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.214556 19844 sys_catalog.cc:564] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: configured and running, proceeding with master startup.
06:59:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.223142 19862 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.223384 19862 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:16.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.220631 19775 master_runner.cc:186] Error getting master registration for 127.19.41.254:38787: OK, instance_id { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" instance_seqno: 1714114755319229 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.227938 19775 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.246665 19700 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 3
06:59:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.258317 19700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: Bootstrap starting.
06:59:16.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.262384 19700 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: Neither blocks nor log segments found. Creating new log.
06:59:16.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.263736 19700 log.cc:824] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: Log is configured to *not* fsync() on all Append() calls
06:59:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.266698 19700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: No bootstrap required, opened a new log
06:59:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.275864 19700 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.276203 19700 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.276338 19700 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b1b5e28af8e4dd098b8fd9d26e492e6, State: Initialized, Role: FOLLOWER
06:59:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.277088 19700 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.277928 19865 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.278200 19865 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.279296 19700 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: configured and running, proceeding with master startup.
06:59:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.289821 19876 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.290118 19876 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.289175 19634 master_runner.cc:186] Error getting master registration for 127.19.41.253:43055: OK, instance_id { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" instance_seqno: 1714114755746024 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:16.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.299387 19634 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:16.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.309661 19654 tablet.cc:2369] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:16.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.394639 19798 tablet.cc:2369] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.442375 19773 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } attempt: 5
06:59:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.455260 19773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: Bootstrap starting.
06:59:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.459213 19773 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: Neither blocks nor log segments found. Creating new log.
06:59:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.460502 19773 log.cc:824] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: Log is configured to *not* fsync() on all Append() calls
06:59:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.463490 19773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: No bootstrap required, opened a new log
06:59:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.472642 19773 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.472982 19773 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.473093 19773 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5671858a5f3047cfbe16b39141fdcf33, State: Initialized, Role: FOLLOWER
06:59:16.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.473855 19773 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.474923 19879 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.475399 19879 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.476132 19773 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: configured and running, proceeding with master startup.
06:59:16.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.485739 19727 tablet.cc:2369] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.492864 19865 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.493196 19865 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.495245 19865 leader_election.cc:288] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5671858a5f3047cfbe16b39141fdcf33 (127.19.41.253:43055), 300b834a891d4ae9b17585f0a7481813 (127.19.41.252:38057)
06:59:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.495748 19704 master_runner.cc:263] No leader master found from master 5671858a5f3047cfbe16b39141fdcf33
06:59:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.495877 19704 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.496444 19748 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5671858a5f3047cfbe16b39141fdcf33" is_pre_election: true
06:59:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.496966 19748 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9b1b5e28af8e4dd098b8fd9d26e492e6 in term 0.
06:59:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.497725 19650 leader_election.cc:302] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5671858a5f3047cfbe16b39141fdcf33, 9b1b5e28af8e4dd098b8fd9d26e492e6; no voters: 
06:59:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.498723 19819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "300b834a891d4ae9b17585f0a7481813" is_pre_election: true
06:59:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.499192 19819 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9b1b5e28af8e4dd098b8fd9d26e492e6 in term 0.
06:59:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.500094 19865 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:59:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.500242 19865 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:59:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.500344 19865 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 0 FOLLOWER]: Advancing to term 1
06:59:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.503527 19865 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.505033 19748 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5671858a5f3047cfbe16b39141fdcf33"
06:59:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.505230 19748 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 0 FOLLOWER]: Advancing to term 1
06:59:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.508517 19748 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9b1b5e28af8e4dd098b8fd9d26e492e6 in term 1.
06:59:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.509963 19650 leader_election.cc:302] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5671858a5f3047cfbe16b39141fdcf33, 9b1b5e28af8e4dd098b8fd9d26e492e6; no voters: 
06:59:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.512542 19865 leader_election.cc:288] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [CANDIDATE]: Term 1 election: Requested vote from peers 5671858a5f3047cfbe16b39141fdcf33 (127.19.41.253:43055), 300b834a891d4ae9b17585f0a7481813 (127.19.41.252:38057)
06:59:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.513111 19865 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 1 FOLLOWER]: Leader election won for term 1
06:59:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.513585 19865 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [term 1 LEADER]: Becoming Leader. State: Replica: 9b1b5e28af8e4dd098b8fd9d26e492e6, State: Running, Role: LEADER
06:59:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.514391 19865 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } }
06:59:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.524964 19819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "300b834a891d4ae9b17585f0a7481813"
06:59:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.525188 19819 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 0 FOLLOWER]: Advancing to term 1
06:59:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.528563 19819 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9b1b5e28af8e4dd098b8fd9d26e492e6 in term 1.
06:59:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.533733 19892 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9b1b5e28af8e4dd098b8fd9d26e492e6. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.534184 19892 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: This master's current role is: LEADER
06:59:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.538661 19890 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:16.538897 19890 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:16.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.542418 19894 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:59:16.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.545109 19894 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:59:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.565837 19819 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [term 1 FOLLOWER]: Refusing update from remote peer 9b1b5e28af8e4dd098b8fd9d26e492e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:16.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.566936 19892 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.572368 19748 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [term 1 FOLLOWER]: Refusing update from remote peer 9b1b5e28af8e4dd098b8fd9d26e492e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.573386 19892 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.594657 19879 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9b1b5e28af8e4dd098b8fd9d26e492e6. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.594970 19879 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.599910 19895 mvcc.cc:205] Tried to move back new op lower bound from 7021014042881966080 to 7021014042695147520. Current Snapshot: MvccSnapshot[applied={T|T < 7021014042881966080}]
06:59:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.605562 19851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9b1b5e28af8e4dd098b8fd9d26e492e6. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.605854 19851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.606452 19897 mvcc.cc:205] Tried to move back new op lower bound from 7021014042881966080 to 7021014042695147520. Current Snapshot: MvccSnapshot[applied={T|T < 7021014042881966080}]
06:59:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.607021 19865 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.607350 19865 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: This master's current role is: LEADER
06:59:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.609023 19894 catalog_manager.cc:1347] Generated new cluster ID: b5a323143c6f4ce59aa85c3eaf7bd0f8
06:59:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.609120 19894 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:59:16.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.611680 19896 mvcc.cc:205] Tried to move back new op lower bound from 7021014042881966080 to 7021014042695147520. Current Snapshot: MvccSnapshot[applied={T|T < 7021014042881966080}]
06:59:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.618695 19892 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.619693 19892 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6 [sys.catalog]: This master's current role is: LEADER
06:59:16.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.622917 19879 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.623128 19879 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.635129 19851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b1b5e28af8e4dd098b8fd9d26e492e6" member_type: VOTER last_known_addr { host: "127.19.41.254" port: 38787 } } peers { permanent_uuid: "5671858a5f3047cfbe16b39141fdcf33" member_type: VOTER last_known_addr { host: "127.19.41.253" port: 43055 } } peers { permanent_uuid: "300b834a891d4ae9b17585f0a7481813" member_type: VOTER last_known_addr { host: "127.19.41.252" port: 38057 } } }
06:59:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.635355 19851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813 [sys.catalog]: This master's current role is: FOLLOWER
06:59:16.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.649432 19894 catalog_manager.cc:1370] Generated new certificate authority record
06:59:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.651162 19894 catalog_manager.cc:1504] Loading token signing keys...
06:59:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.671862 19894 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 9b1b5e28af8e4dd098b8fd9d26e492e6: Generated new TSK 0
06:59:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.673391 19894 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.707556 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/logs
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data/info.pb
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.193:0
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.193
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.193
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.045955 19904 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.046212 19904 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.046298 19904 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.046362 19904 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.053684 19904 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.053807 19904 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.053858 19904 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.053928 19904 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.41.193
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.062856 19904 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.193:0
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data/info.pb
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.193
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.193
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/logs
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:17.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.064045 19904 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:17.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.065634 19904 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:17.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.077203 19909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.077976 19912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.078823 19910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.079627 19904 server_base.cc:1034] running on GCE node
06:59:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.080222 19904 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:17.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.081038 19904 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.082252 19904 hybrid_clock.cc:648] HybridClock initialized: now 1714114757082209 us; error 44 us; skew 500 ppm
06:59:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.082502 19904 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.084633 19904 webserver.cc:460] Webserver started at http://127.19.41.193:38221/ using document root <none> and password file <none>
06:59:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.085279 19904 fs_manager.cc:359] Metadata directory not provided
06:59:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.085363 19904 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.085604 19904 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.088083 19904 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data/instance:
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6a9171e8c1d44838d03288acc734f59"
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.088793 19904 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal/instance:
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6a9171e8c1d44838d03288acc734f59"
06:59:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.093196 19904 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.006s	sys 0.000s
06:59:17.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.096104 19918 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.097357 19904 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.097513 19904 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6a9171e8c1d44838d03288acc734f59"
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.097726 19904 fs_report.cc:383] FS layout report
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data/data
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.111946 19904 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.112821 19904 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.113035 19904 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:17.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.114014 19904 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.115479 19904 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.115564 19904 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.115629 19904 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.115684 19904 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.144811 19904 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.193:35953
06:59:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.144865 20030 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.193:35953 every 8 connection(s)
06:59:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.145949 19904 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/data/info.pb
06:59:17.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.148574 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 19904
06:59:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.148797 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-0/wal/instance
06:59:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.152799 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/logs
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data/info.pb
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.194:0
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.194
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.194
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.162480 20033 heartbeater.cc:347] Connected to a master server at 127.19.41.253:43055
06:59:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.162483 20032 heartbeater.cc:347] Connected to a master server at 127.19.41.252:38057
06:59:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.162899 20031 heartbeater.cc:347] Connected to a master server at 127.19.41.254:38787
06:59:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.162901 20032 heartbeater.cc:464] Registering TS with master...
06:59:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.163048 20031 heartbeater.cc:464] Registering TS with master...
06:59:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.162896 20033 heartbeater.cc:464] Registering TS with master...
06:59:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.164139 20031 heartbeater.cc:511] Master 127.19.41.254:38787 requested a full tablet report, sending...
06:59:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.164276 20032 heartbeater.cc:511] Master 127.19.41.252:38057 requested a full tablet report, sending...
06:59:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.164420 20033 heartbeater.cc:511] Master 127.19.41.253:43055 requested a full tablet report, sending...
06:59:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.166492 19665 ts_manager.cc:194] Registered new tserver with Master: f6a9171e8c1d44838d03288acc734f59 (127.19.41.193:35953)
06:59:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.166656 19809 ts_manager.cc:194] Registered new tserver with Master: f6a9171e8c1d44838d03288acc734f59 (127.19.41.193:35953)
06:59:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.166659 19738 ts_manager.cc:194] Registered new tserver with Master: f6a9171e8c1d44838d03288acc734f59 (127.19.41.193:35953)
06:59:17.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.168843 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.41.193:39931
06:59:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.225320 19862 catalog_manager.cc:1259] Loaded cluster ID: b5a323143c6f4ce59aa85c3eaf7bd0f8
06:59:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.225433 19862 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: loading cluster ID for follower catalog manager: success
06:59:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.227911 19862 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: acquiring CA information for follower catalog manager: success
06:59:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.229418 19862 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 300b834a891d4ae9b17585f0a7481813: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.479619 20041 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.479884 20041 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.479990 20041 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.480055 20041 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.487378 20041 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.487529 20041 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.487598 20041 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.487721 20041 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.41.194
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.496357 20041 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.194:0
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data/info.pb
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.194
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.194
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/logs
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.497787 20041 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:17.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.499529 20041 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:17.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.511482 20047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.512372 20049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.513084 20046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.513895 20041 server_base.cc:1034] running on GCE node
06:59:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.514421 20041 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.515278 20041 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.516458 20041 hybrid_clock.cc:648] HybridClock initialized: now 1714114757516421 us; error 42 us; skew 500 ppm
06:59:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.516702 20041 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.518841 20041 webserver.cc:460] Webserver started at http://127.19.41.194:38487/ using document root <none> and password file <none>
06:59:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.519470 20041 fs_manager.cc:359] Metadata directory not provided
06:59:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.519552 20041 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.519794 20041 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.522159 20041 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data/instance:
06:59:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eef349bbf8d24181926900a522fe9624"
06:59:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.522902 20041 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal/instance:
06:59:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eef349bbf8d24181926900a522fe9624"
06:59:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.527532 20041 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.004s	sys 0.000s
06:59:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.530529 20055 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.531809 20041 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.531936 20041 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eef349bbf8d24181926900a522fe9624"
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.532141 20041 fs_report.cc:383] FS layout report
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data/data
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.540745 19890 catalog_manager.cc:1259] Loaded cluster ID: b5a323143c6f4ce59aa85c3eaf7bd0f8
06:59:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.540830 19890 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: loading cluster ID for follower catalog manager: success
06:59:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.543298 19890 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: acquiring CA information for follower catalog manager: success
06:59:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.544837 19890 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5671858a5f3047cfbe16b39141fdcf33: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.556870 20041 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.557637 20041 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.557869 20041 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.558768 20041 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.560335 20041 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.560433 20041 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.560504 20041 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.560560 20041 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.591370 20041 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.194:34805
06:59:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.591429 20167 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.194:34805 every 8 connection(s)
06:59:17.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.592530 20041 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/data/info.pb
06:59:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.600778 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 20041
06:59:17.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.600978 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-1/wal/instance
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.605432 19623 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/logs
06:59:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data/info.pb
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.195:0
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.195
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.195
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:17.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.608201 20169 heartbeater.cc:347] Connected to a master server at 127.19.41.252:38057
06:59:17.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.608461 20169 heartbeater.cc:464] Registering TS with master...
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.609472 20169 heartbeater.cc:511] Master 127.19.41.252:38057 requested a full tablet report, sending...
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.610173 20168 heartbeater.cc:347] Connected to a master server at 127.19.41.254:38787
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.610292 20168 heartbeater.cc:464] Registering TS with master...
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.610641 20170 heartbeater.cc:347] Connected to a master server at 127.19.41.253:43055
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.610788 20170 heartbeater.cc:464] Registering TS with master...
06:59:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.610841 20168 heartbeater.cc:511] Master 127.19.41.254:38787 requested a full tablet report, sending...
06:59:17.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.611238 19809 ts_manager.cc:194] Registered new tserver with Master: eef349bbf8d24181926900a522fe9624 (127.19.41.194:34805)
06:59:17.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.611335 20170 heartbeater.cc:511] Master 127.19.41.253:43055 requested a full tablet report, sending...
06:59:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.611896 19665 ts_manager.cc:194] Registered new tserver with Master: eef349bbf8d24181926900a522fe9624 (127.19.41.194:34805)
06:59:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.612327 19738 ts_manager.cc:194] Registered new tserver with Master: eef349bbf8d24181926900a522fe9624 (127.19.41.194:34805)
06:59:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.612910 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.41.194:42175
06:59:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.918334 20175 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.918644 20175 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.918766 20175 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.918866 20175 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.926141 20175 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.926304 20175 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.926374 20175 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.926494 20175 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.41.195
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.935496 20175 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.41.212:36265
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data
06:59:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.41.195:0
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data/info.pb
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.41.195
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.41.254:38787,127.19.41.253:43055,127.19.41.252:38057
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.41.195
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/logs
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
06:59:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:59:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.937038 20175 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.938757 20175 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.950497 20181 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.951578 20180 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.952569 20183 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.953874 20175 server_base.cc:1034] running on GCE node
06:59:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.954766 20175 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:17.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.955559 20175 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.956750 20175 hybrid_clock.cc:648] HybridClock initialized: now 1714114757956696 us; error 46 us; skew 500 ppm
06:59:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.957037 20175 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.959107 20175 webserver.cc:460] Webserver started at http://127.19.41.195:34937/ using document root <none> and password file <none>
06:59:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.959787 20175 fs_manager.cc:359] Metadata directory not provided
06:59:17.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.959889 20175 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.960239 20175 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.962652 20175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data/instance:
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed56e2aba6294be692849dc10e8763f9"
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.963446 20175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal/instance:
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed56e2aba6294be692849dc10e8763f9"
06:59:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.968153 20175 fs_manager.cc:691] Time spent creating directory manager: real 0.004s	user 0.004s	sys 0.000s
06:59:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.971807 20189 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.973059 20175 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.973218 20175 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data,/tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed56e2aba6294be692849dc10e8763f9"
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:17 on dist-test-slave-7lpv"
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.973420 20175 fs_report.cc:383] FS layout report
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data/data
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:17.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:17.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.986407 20175 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.987171 20175 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.987460 20175 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.988368 20175 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.989789 20175 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.989892 20175 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.990000 20175 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.990089 20175 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.019496 20175 rpc_server.cc:307] RPC server started. Bound to: 127.19.41.195:38083
06:59:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.019562 20301 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.41.195:38083 every 8 connection(s)
06:59:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.020694 20175 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/data/info.pb
06:59:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.023968 19623 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu as pid 20175
06:59:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.024152 19623 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ9V9WF/test-tmp/mini-kudu-cluster2925830934186423423/ts-2/wal/instance
06:59:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.035970 20302 heartbeater.cc:347] Connected to a master server at 127.19.41.254:38787
06:59:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.036293 20302 heartbeater.cc:464] Registering TS with master...
06:59:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.036398 20304 heartbeater.cc:347] Connected to a master server at 127.19.41.253:43055
06:59:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.036521 20304 heartbeater.cc:464] Registering TS with master...
06:59:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.037554 20304 heartbeater.cc:511] Master 127.19.41.253:43055 requested a full tablet report, sending...
06:59:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.037614 20302 heartbeater.cc:511] Master 127.19.41.254:38787 requested a full tablet report, sending...
06:59:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.038985 20303 heartbeater.cc:347] Connected to a master server at 127.19.41.252:38057
06:59:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.039115 20303 heartbeater.cc:464] Registering TS with master...
06:59:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.039359 19665 ts_manager.cc:194] Registered new tserver with Master: ed56e2aba6294be692849dc10e8763f9 (127.19.41.195:38083)
06:59:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.039377 19738 ts_manager.cc:194] Registered new tserver with Master: ed56e2aba6294be692849dc10e8763f9 (127.19.41.195:38083)
06:59:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.039667 20303 heartbeater.cc:511] Master 127.19.41.252:38057 requested a full tablet report, sending...
06:59:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.040328 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.41.195:41501
06:59:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.040520 19809 ts_manager.cc:194] Registered new tserver with Master: ed56e2aba6294be692849dc10e8763f9 (127.19.41.195:38083)
06:59:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.049263 19623 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:59:18.051 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:59:18.051 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

06:59:18.052 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.19.41.254"
      port: 38787
    }
    bound_http_address {
      host: "127.19.41.254"
      port: 37683
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.19.41.253"
      port: 43055
    }
    bound_http_address {
      host: "127.19.41.253"
      port: 40225
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.19.41.252"
      port: 38057
    }
    bound_http_address {
      host: "127.19.41.252"
      port: 35431
    }
  }
}

06:59:18.053 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

06:59:18.053 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.19.41.193"
      port: 35953
    }
    bound_http_address {
      host: "127.19.41.193"
      port: 38221
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.19.41.194"
      port: 34805
    }
    bound_http_address {
      host: "127.19.41.194"
      port: 38487
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.19.41.195"
      port: 38083
    }
    bound_http_address {
      host: "127.19.41.195"
      port: 34937
    }
  }
}

06:59:18.054 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:59:18.056 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:59:18.057 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
06:59:18.058 [INFO - main] (Logging.scala:57) ==============================================================
06:59:18.058 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
06:59:18.058 [INFO - main] (Logging.scala:57) ==============================================================
06:59:18.058 [INFO - main] (Logging.scala:57) Submitted application: test
06:59:18.059 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
06:59:18.060 [INFO - main] (Logging.scala:57) Limiting resource is cpu
06:59:18.060 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
06:59:18.061 [INFO - main] (Logging.scala:57) Changing view acls to: slave
06:59:18.061 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
06:59:18.061 [INFO - main] (Logging.scala:57) Changing view acls groups to: 
06:59:18.061 [INFO - main] (Logging.scala:57) Changing modify acls groups to: 
06:59:18.061 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(slave); groups with view permissions: Set(); users  with modify permissions: Set(slave); groups with modify permissions: Set()
06:59:18.082 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 35031.
06:59:18.084 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
06:59:18.085 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
06:59:18.085 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
06:59:18.085 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
06:59:18.086 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
06:59:18.086 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-c473def6-d863-4a6f-bd0d-6fae40e78566
06:59:18.087 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
06:59:18.088 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
06:59:18.106 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-7lpv.c.gcp-upstream.internal
06:59:18.110 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40993.
06:59:18.110 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-7lpv.c.gcp-upstream.internal:40993
06:59:18.110 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
06:59:18.110 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-7lpv.c.gcp-upstream.internal, 40993, None)
06:59:18.111 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-7lpv.c.gcp-upstream.internal:40993 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-7lpv.c.gcp-upstream.internal, 40993, None)
06:59:18.112 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-7lpv.c.gcp-upstream.internal, 40993, None)
06:59:18.112 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-7lpv.c.gcp-upstream.internal, 40993, None)
06:59:18.120 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:59:18.128 [DEBUG - main] (Connection.java:748) [peer master-127.19.41.253:43055(127.19.41.253:43055)] connecting to peer
06:59:18.130 [DEBUG - main] (Connection.java:748) [peer master-127.19.41.254:38787(127.19.41.254:38787)] connecting to peer
06:59:18.131 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.41.253:43055(127.19.41.253:43055)] Successfully connected to peer
06:59:18.131 [DEBUG - main] (Connection.java:748) [peer master-127.19.41.252:38057(127.19.41.252:38057)] connecting to peer
06:59:18.133 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.41.254:38787(127.19.41.254:38787)] Successfully connected to peer
06:59:18.135 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.41.253
06:59:18.136 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.41.254
06:59:18.138 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.41.252:38057(127.19.41.252:38057)] Successfully connected to peer
06:59:18.140 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.41.252
06:59:18.147 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:18.155 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcffc4245, L:/127.0.0.1:57502 - R:/127.19.41.254:38787] using SASL/PLAIN
06:59:18.155 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:18.158 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:59:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.169494 19665 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57502:
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c1_i"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c2_s"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c3_double"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DOUBLE
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.172 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c4_long"
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
06:59:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c5_bool"
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c6_short"
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT16
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c7_float"
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: FLOAT
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c8_binary"
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BINARY
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c9_unixtime_micros"
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: UNIXTIME_MICROS
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c10_byte"
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c11_decimal32"
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL32
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 9
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c12_decimal64"
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL64
06:59:18.175 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf8993252, L:/127.0.0.1:57306 - R:/127.19.41.252:38057] using SASL/PLAIN
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 18
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c13_decimal128"
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL128
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 38
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c14_varchar"
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: VARCHAR
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       length: 65535
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c15_date"
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DATE
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
06:59:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.171592 20031 heartbeater.cc:503] Master 127.19.41.254:38787 was elected leader, sending a full tablet report...
06:59:18.178 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1c3bf00c, L:/127.0.0.1:54138 - R:/127.19.41.253:43055] using SASL/PLAIN
06:59:18.202 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table b9aff82c00cd4880bb8c46eca8c93d5c
06:59:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.227362 19966 tablet_service.cc:1467] Processing CreateTablet for tablet 7ecde5f8f3b44d869d288f35539e0f00 (DEFAULT_TABLE table=test [id=b9aff82c00cd4880bb8c46eca8c93d5c]), partition=RANGE (key) PARTITION VALUES < 50
06:59:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.227365 20103 tablet_service.cc:1467] Processing CreateTablet for tablet dbfe916efc16446abee11dc3c5524278 (DEFAULT_TABLE table=test [id=b9aff82c00cd4880bb8c46eca8c93d5c]), partition=RANGE (key) PARTITION 50 <= VALUES
06:59:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.228682 19966 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ecde5f8f3b44d869d288f35539e0f00. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:18.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.228741 20103 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dbfe916efc16446abee11dc3c5524278. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:18.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.238111 20356 tablet_bootstrap.cc:492] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: Bootstrap starting.
06:59:18.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.238126 20357 tablet_bootstrap.cc:492] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: Bootstrap starting.
06:59:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.240887 20357 tablet_bootstrap.cc:654] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: Neither blocks nor log segments found. Creating new log.
06:59:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.240890 20356 tablet_bootstrap.cc:654] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: Neither blocks nor log segments found. Creating new log.
06:59:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.241763 20357 log.cc:824] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: Log is configured to *not* fsync() on all Append() calls
06:59:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.241770 20356 log.cc:824] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: Log is configured to *not* fsync() on all Append() calls
06:59:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.243997 20356 tablet_bootstrap.cc:492] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: No bootstrap required, opened a new log
06:59:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.244000 20357 tablet_bootstrap.cc:492] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: No bootstrap required, opened a new log
06:59:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.244200 20357 ts_tablet_manager.cc:1367] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: Time spent bootstrapping tablet: real 0.006s	user 0.004s	sys 0.000s
06:59:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.244242 20356 ts_tablet_manager.cc:1367] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: Time spent bootstrapping tablet: real 0.006s	user 0.000s	sys 0.004s
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.249876 20357 raft_consensus.cc:348] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f6a9171e8c1d44838d03288acc734f59" member_type: VOTER last_known_addr { host: "127.19.41.193" port: 35953 } }
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.249943 20356 raft_consensus.cc:348] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "eef349bbf8d24181926900a522fe9624" member_type: VOTER last_known_addr { host: "127.19.41.194" port: 34805 } }
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250207 20357 raft_consensus.cc:374] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250207 20356 raft_consensus.cc:374] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250293 20357 raft_consensus.cc:729] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f6a9171e8c1d44838d03288acc734f59, State: Initialized, Role: FOLLOWER
06:59:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250298 20356 raft_consensus.cc:729] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eef349bbf8d24181926900a522fe9624, State: Initialized, Role: FOLLOWER
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250986 20357 consensus_queue.cc:260] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f6a9171e8c1d44838d03288acc734f59" member_type: VOTER last_known_addr { host: "127.19.41.193" port: 35953 } }
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.250989 20356 consensus_queue.cc:260] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "eef349bbf8d24181926900a522fe9624" member_type: VOTER last_known_addr { host: "127.19.41.194" port: 34805 } }
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251173 20356 raft_consensus.cc:388] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251181 20357 raft_consensus.cc:388] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251248 20356 raft_consensus.cc:482] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251295 20357 raft_consensus.cc:482] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251345 20356 raft_consensus.cc:3037] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 0 FOLLOWER]: Advancing to term 1
06:59:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.251421 20357 raft_consensus.cc:3037] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 0 FOLLOWER]: Advancing to term 1
06:59:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.253074 20356 raft_consensus.cc:504] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "eef349bbf8d24181926900a522fe9624" member_type: VOTER last_known_addr { host: "127.19.41.194" port: 34805 } }
06:59:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.253285 20357 raft_consensus.cc:504] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f6a9171e8c1d44838d03288acc734f59" member_type: VOTER last_known_addr { host: "127.19.41.193" port: 35953 } }
06:59:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.253448 20356 leader_election.cc:302] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: eef349bbf8d24181926900a522fe9624; no voters: 
06:59:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.253716 20357 leader_election.cc:302] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f6a9171e8c1d44838d03288acc734f59; no voters: 
06:59:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.253974 20356 leader_election.cc:288] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.254137 20360 raft_consensus.cc:2781] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 1 FOLLOWER]: Leader election won for term 1
06:59:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.254366 20357 leader_election.cc:288] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.254529 20361 raft_consensus.cc:2781] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 1 FOLLOWER]: Leader election won for term 1
06:59:18.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.261461 20360 raft_consensus.cc:686] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [term 1 LEADER]: Becoming Leader. State: Replica: eef349bbf8d24181926900a522fe9624, State: Running, Role: LEADER
06:59:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.262310 20356 ts_tablet_manager.cc:1398] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624: Time spent starting tablet: real 0.018s	user 0.008s	sys 0.010s
06:59:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.262864 20168 heartbeater.cc:503] Master 127.19.41.254:38787 was elected leader, sending a full tablet report...
06:59:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.262220 20360 consensus_queue.cc:237] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "eef349bbf8d24181926900a522fe9624" member_type: VOTER last_known_addr { host: "127.19.41.194" port: 34805 } }
06:59:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.263818 20361 raft_consensus.cc:686] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [term 1 LEADER]: Becoming Leader. State: Replica: f6a9171e8c1d44838d03288acc734f59, State: Running, Role: LEADER
06:59:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.264005 20357 ts_tablet_manager.cc:1398] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59: Time spent starting tablet: real 0.020s	user 0.014s	sys 0.006s
06:59:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.264766 20361 consensus_queue.cc:237] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f6a9171e8c1d44838d03288acc734f59" member_type: VOTER last_known_addr { host: "127.19.41.193" port: 35953 } }
06:59:18.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.269292 19665 catalog_manager.cc:5484] T dbfe916efc16446abee11dc3c5524278 P eef349bbf8d24181926900a522fe9624 reported cstate change: term changed from 0 to 1, leader changed from <none> to eef349bbf8d24181926900a522fe9624 (127.19.41.194). New cstate: current_term: 1 leader_uuid: "eef349bbf8d24181926900a522fe9624" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "eef349bbf8d24181926900a522fe9624" member_type: VOTER last_known_addr { host: "127.19.41.194" port: 34805 } health_report { overall_health: HEALTHY } } }
06:59:18.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.295022 19664 catalog_manager.cc:5484] T 7ecde5f8f3b44d869d288f35539e0f00 P f6a9171e8c1d44838d03288acc734f59 reported cstate change: term changed from 0 to 1, leader changed from <none> to f6a9171e8c1d44838d03288acc734f59 (127.19.41.193). New cstate: current_term: 1 leader_uuid: "f6a9171e8c1d44838d03288acc734f59" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f6a9171e8c1d44838d03288acc734f59" member_type: VOTER last_known_addr { host: "127.19.41.193" port: 35953 } health_report { overall_health: HEALTHY } } }
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.332765 19664 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57502:
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "val"
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:18.344 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 705b0d90d6a645b59e85da93de7e80ab
06:59:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.345252 20171 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:dbfe916efc16446abee11dc3c5524278)
06:59:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.355341 20237 tablet_service.cc:1467] Processing CreateTablet for tablet 24bc17d328574140bb873ed6b06bc13d (DEFAULT_TABLE table=simple-test [id=705b0d90d6a645b59e85da93de7e80ab]), partition=RANGE (key) PARTITION UNBOUNDED
06:59:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.356487 20237 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24bc17d328574140bb873ed6b06bc13d. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:18.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.365437 20371 tablet_bootstrap.cc:492] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: Bootstrap starting.
06:59:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.368016 20371 tablet_bootstrap.cc:654] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: Neither blocks nor log segments found. Creating new log.
06:59:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.368940 20371 log.cc:824] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: Log is configured to *not* fsync() on all Append() calls
06:59:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.371287 20371 tablet_bootstrap.cc:492] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: No bootstrap required, opened a new log
06:59:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.371486 20371 ts_tablet_manager.cc:1367] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: Time spent bootstrapping tablet: real 0.006s	user 0.004s	sys 0.001s
06:59:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.376971 20371 raft_consensus.cc:348] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ed56e2aba6294be692849dc10e8763f9" member_type: VOTER last_known_addr { host: "127.19.41.195" port: 38083 } }
06:59:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.377272 20371 raft_consensus.cc:374] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.377367 20371 raft_consensus.cc:729] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed56e2aba6294be692849dc10e8763f9, State: Initialized, Role: FOLLOWER
06:59:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.377892 20371 consensus_queue.cc:260] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ed56e2aba6294be692849dc10e8763f9" member_type: VOTER last_known_addr { host: "127.19.41.195" port: 38083 } }
06:59:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.378096 20371 raft_consensus.cc:388] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.378206 20371 raft_consensus.cc:482] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.378319 20371 raft_consensus.cc:3037] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 0 FOLLOWER]: Advancing to term 1
06:59:18.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.380367 20371 raft_consensus.cc:504] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ed56e2aba6294be692849dc10e8763f9" member_type: VOTER last_known_addr { host: "127.19.41.195" port: 38083 } }
06:59:18.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.380760 20371 leader_election.cc:302] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ed56e2aba6294be692849dc10e8763f9; no voters: 
06:59:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.381302 20371 leader_election.cc:288] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.381449 20373 raft_consensus.cc:2781] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 1 FOLLOWER]: Leader election won for term 1
06:59:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.382457 20302 heartbeater.cc:503] Master 127.19.41.254:38787 was elected leader, sending a full tablet report...
06:59:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.382715 20373 raft_consensus.cc:686] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [term 1 LEADER]: Becoming Leader. State: Replica: ed56e2aba6294be692849dc10e8763f9, State: Running, Role: LEADER
06:59:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.383196 20373 consensus_queue.cc:237] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ed56e2aba6294be692849dc10e8763f9" member_type: VOTER last_known_addr { host: "127.19.41.195" port: 38083 } }
06:59:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.383335 20371 ts_tablet_manager.cc:1398] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9: Time spent starting tablet: real 0.012s	user 0.004s	sys 0.008s
06:59:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.387626 19664 catalog_manager.cc:5484] T 24bc17d328574140bb873ed6b06bc13d P ed56e2aba6294be692849dc10e8763f9 reported cstate change: term changed from 0 to 1, leader changed from <none> to ed56e2aba6294be692849dc10e8763f9 (127.19.41.195). New cstate: current_term: 1 leader_uuid: "ed56e2aba6294be692849dc10e8763f9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ed56e2aba6294be692849dc10e8763f9" member_type: VOTER last_known_addr { host: "127.19.41.195" port: 38083 } health_report { overall_health: HEALTHY } } }
06:59:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.399399 20034 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:7ecde5f8f3b44d869d288f35539e0f00)
06:59:18.413 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 7ecde5f8f3b44d869d288f35539e0f00 for table 'test' with partition [<start>, 0x80000032)
06:59:18.413 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:18.413 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=7ecde5f8f3b44d869d288f35539e0f00@[f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:18.413 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=null) after lookup
06:59:18.414 [DEBUG - kudu-nio-1] (Connection.java:748) [peer f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)] connecting to peer
06:59:18.415 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)] Successfully connected to peer
06:59:18.424 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:18.427 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2a44102b, L:/127.0.0.1:36054 - R:/127.19.41.193:35953] using TOKEN/null
06:59:18.467 [INFO - main] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-taskQ9V9WF/java/build/dist-test/spark-warehouse').
06:59:18.467 [INFO - main] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-taskQ9V9WF/java/build/dist-test/spark-warehouse'.
06:59:18.475 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table b9aff82c00cd4880bb8c46eca8c93d5c
06:59:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.527359 20305 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:24bc17d328574140bb873ed6b06bc13d)
06:59:18.790 [DEBUG - main] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:18.792 [DEBUG - kudu-nio-0] (Connection.java:757) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] Successfully connected to peer
06:59:18.803 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:18.805 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1cb69157, L:/127.0.0.1:35344 - R:/127.19.41.194:34805] using TOKEN/null
06:59:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.900871 20034 maintenance_manager.cc:419] P f6a9171e8c1d44838d03288acc734f59: Scheduling FlushMRSOp(7ecde5f8f3b44d869d288f35539e0f00): perf score=1.000000
06:59:18.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.983680 19923 maintenance_manager.cc:643] P f6a9171e8c1d44838d03288acc734f59: FlushMRSOp(7ecde5f8f3b44d869d288f35539e0f00) complete. Timing: real 0.082s	user 0.060s	sys 0.009s Metrics: {"bytes_written":39815,"cfile_init":1,"compiler_manager_pool.queue_time_us":421,"dirs.queue_time_us":404,"dirs.run_cpu_time_us":220,"dirs.run_wall_time_us":1082,"drs_written":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1800,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":50,"thread_start_us":517,"threads_started":2}
06:59:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.985080 20034 maintenance_manager.cc:419] P f6a9171e8c1d44838d03288acc734f59: Scheduling UndoDeltaBlockGCOp(7ecde5f8f3b44d869d288f35539e0f00): 1549 bytes on disk
06:59:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.986117 19923 maintenance_manager.cc:643] P f6a9171e8c1d44838d03288acc734f59: UndoDeltaBlockGCOp(7ecde5f8f3b44d869d288f35539e0f00) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
06:59:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.097427 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.000000
06:59:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.220270 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.122s	user 0.077s	sys 0.015s Metrics: {"bytes_written":39933,"cfile_init":1,"compiler_manager_pool.queue_time_us":2869,"dirs.queue_time_us":6453,"dirs.run_cpu_time_us":199,"dirs.run_wall_time_us":10625,"drs_written":1,"lbm_read_time_us":123,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3522,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":53,"thread_start_us":9343,"threads_started":2}
06:59:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.221735 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 729 bytes on disk
06:59:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.223011 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":123,"lbm_reads_lt_1ms":4}
06:59:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.473942 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.578738 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.105s	user 0.064s	sys 0.004s Metrics: {"bytes_written":40068,"cfile_init":1,"dirs.queue_time_us":49,"dirs.run_cpu_time_us":198,"dirs.run_wall_time_us":1324,"drs_written":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3093,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":64}
06:59:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.579677 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 849 bytes on disk
06:59:19.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.580520 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":91,"lbm_reads_lt_1ms":4}
06:59:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.831436 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.881772 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.050s	user 0.045s	sys 0.003s Metrics: {"bytes_written":40137,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":224,"dirs.run_wall_time_us":1551,"drs_written":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2134,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":60}
06:59:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.882609 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 805 bytes on disk
06:59:19.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.883276 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4}
06:59:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.134109 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.185146 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.051s	user 0.044s	sys 0.004s Metrics: {"bytes_written":40313,"cfile_init":1,"dirs.queue_time_us":58,"dirs.run_cpu_time_us":177,"dirs.run_wall_time_us":1309,"drs_written":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1999,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":70,"spinlock_wait_cycles":22912}
06:59:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.185736 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 918 bytes on disk
06:59:20.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.186342 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4}
06:59:20.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.437876 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.495193 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.057s	user 0.053s	sys 0.004s Metrics: {"bytes_written":40416,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":194,"dirs.run_wall_time_us":1769,"drs_written":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2266,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":74}
06:59:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.496052 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 959 bytes on disk
06:59:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.496793 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4}
06:59:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.747704 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.806288 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.058s	user 0.053s	sys 0.000s Metrics: {"bytes_written":40275,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":243,"dirs.run_wall_time_us":1560,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1967,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":80}
06:59:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.806870 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 1022 bytes on disk
06:59:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.807467 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4}
06:59:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.058209 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.115130 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.057s	user 0.053s	sys 0.004s Metrics: {"bytes_written":40455,"cfile_init":1,"dirs.queue_time_us":47,"dirs.run_cpu_time_us":218,"dirs.run_wall_time_us":1589,"drs_written":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2035,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":81}
06:59:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.115962 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 1036 bytes on disk
06:59:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.116534 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":80,"lbm_reads_lt_1ms":4}
06:59:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.367380 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953555
06:59:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.420755 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.053s	user 0.051s	sys 0.000s Metrics: {"bytes_written":40487,"cfile_init":1,"dirs.queue_time_us":65,"dirs.run_cpu_time_us":240,"dirs.run_wall_time_us":1481,"drs_written":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1609,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":82}
06:59:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.421352 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 1048 bytes on disk
06:59:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.421900 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4}
06:59:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.672757 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953555
06:59:21.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.728128 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.055s	user 0.050s	sys 0.004s Metrics: {"bytes_written":40456,"cfile_init":1,"dirs.queue_time_us":59,"dirs.run_cpu_time_us":189,"dirs.run_wall_time_us":1294,"drs_written":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1888,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":78}
06:59:21.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.728899 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 1004 bytes on disk
06:59:21.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.729580 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":84,"lbm_reads_lt_1ms":4}
06:59:21.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.980496 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953555
06:59:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.029963 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.049s	user 0.045s	sys 0.000s Metrics: {"bytes_written":40554,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":211,"dirs.run_wall_time_us":1540,"drs_written":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1742,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":84}
06:59:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.030511 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 1060 bytes on disk
06:59:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.031297 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4}
06:59:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.282153 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953555
06:59:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.335927 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.054s	user 0.051s	sys 0.000s Metrics: {"bytes_written":40434,"cfile_init":1,"dirs.queue_time_us":59,"dirs.run_cpu_time_us":165,"dirs.run_wall_time_us":2142,"drs_written":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1742,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":77}
06:59:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.336656 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 996 bytes on disk
06:59:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.337849 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4}
06:59:22.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.588743 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953466
06:59:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.656957 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.068s	user 0.049s	sys 0.016s Metrics: {"bytes_written":40431,"cfile_init":1,"dirs.queue_time_us":64,"dirs.run_cpu_time_us":199,"dirs.run_wall_time_us":1351,"drs_written":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2419,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":75}
06:59:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.657615 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 968 bytes on disk
06:59:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.658157 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
06:59:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.908967 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling FlushMRSOp(dbfe916efc16446abee11dc3c5524278): perf score=1.953555
06:59:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.952601 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: FlushMRSOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.043s	user 0.037s	sys 0.004s Metrics: {"bytes_written":40162,"cfile_init":1,"dirs.queue_time_us":135,"dirs.run_cpu_time_us":189,"dirs.run_wall_time_us":1427,"drs_written":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1432,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":72}
06:59:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.953310 20171 maintenance_manager.cc:419] P eef349bbf8d24181926900a522fe9624: Scheduling UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278): 926 bytes on disk
06:59:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.953830 20060 maintenance_manager.cc:643] P eef349bbf8d24181926900a522fe9624: UndoDeltaBlockGCOp(dbfe916efc16446abee11dc3c5524278) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
06:59:27.863 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table b9aff82c00cd4880bb8c46eca8c93d5c
06:59:27.931 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for test
06:59:27.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2206) Key ranges for test
06:59:27.931 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, <end>), 35635, "7ecde5f8f3b44d869d288f35539e0f00" [<start>, 0x80000032)
06:59:27.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [<start>, 0x80000066), 35753, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000066, 0x800000A6), 35888, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000A6, 0x800000E2), 35957, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000E2, 0x80000128), 36133, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000128, 0x80000172), 36236, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000172, 0x800001C2), 36095, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800001C2, 0x80000213), 36275, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000213, 0x80000265), 36307, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000265, 0x800002B3), 36276, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800002B3, 0x80000307), 36374, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000307, 0x80000354), 36254, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000354, 0x8000039F), 36251, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000039F, <end>), 35982, "dbfe916efc16446abee11dc3c5524278" [0x80000032, <end>)
06:59:27.946 [INFO - main] (Logging.scala:57) Starting job: collectAsList at SparkSQLTest.scala:471
06:59:27.947 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (collectAsList at SparkSQLTest.scala:471) with 14 output partitions
06:59:27.947 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (collectAsList at SparkSQLTest.scala:471)
06:59:27.947 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
06:59:27.948 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
06:59:27.949 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471), which has no missing parents
06:59:27.955 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 25.2 KiB, free 1993.9 MiB)
06:59:27.957 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 9.6 KiB, free 1993.8 MiB)
06:59:27.958 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-7lpv.c.gcp-upstream.internal:40993 (size: 9.6 KiB, free: 1993.9 MiB)
06:59:27.959 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
06:59:27.960 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 14 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13))
06:59:27.960 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 14 tasks resource profile 0
06:59:27.962 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-7lpv.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5316 bytes) taskResourceAssignments Map()
06:59:27.964 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
06:59:27.972 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:28.027 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 50 rows, more = false, responseScanTimestamp = 7021014089651142656) for scanner KuduScanner(table=test, tablet=7ecde5f8f3b44d869d288f35539e0f00, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:59:28.068 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 17999 bytes result sent to driver
06:59:28.070 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-7lpv.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5322 bytes) taskResourceAssignments Map()
06:59:28.071 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
06:59:28.071 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 109 ms on dist-test-slave-7lpv.c.gcp-upstream.internal (executor driver) (1/14)
06:59:28.076 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:28.117 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 52 rows, more = false, responseScanTimestamp = 7021014090077872128) for scanner KuduScanner(table=test, tablet=dbfe916efc16446abee11dc3c5524278, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x80000066, lastPrimaryKey=<last>)
06:59:28.150 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1). 18525 bytes result sent to driver
06:59:28.152 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-7lpv.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5328 bytes) taskResourceAssignments Map()
06:59:28.153 [INFO - task-result-getter-1] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1) in 83 ms on dist-test-slave-7lpv.c.gcp-upstream.internal (executor driver) (2/14)
06:59:28.153 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
06:59:28.159 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46: runtime error: member call on null pointer of type 'kudu::EncodedKey'
06:59:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #0 0x7f3064516d53 in kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46
06:59:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #1 0x7f306f228048 in kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11
06:59:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #2 0x7f306f22709f in kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3
06:59:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #3 0x7f306f47022d in kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3
06:59:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #4 0x7f30643f4fe3 in kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3
06:59:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #5 0x7f30643eabdc in kudu::InitAndMaybeWrap(std::unique_ptr<kudu::RowwiseIterator, std::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3
06:59:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #6 0x7f30643f1f41 in kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5
06:59:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #7 0x7f30643f0f0d in kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3
06:59:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #8 0x7f306f07f00d in kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3
06:59:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #9 0x7f3071b7020d in kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15
06:59:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #10 0x7f3071b6a40b in kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16
06:59:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #11 0x7f306c302556 in kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/asan/src/kudu/tserver/tserver_service.service.cc:263:13
06:59:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #12 0x7f306c302285 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
06:59:29.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #13 0x7f306933e4fe in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
06:59:29.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #14 0x7f306933d3de in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3
06:59:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #15 0x7f30693438b5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15
06:59:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #16 0x7f30693441df in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5
06:59:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #17 0x7f3069344008 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
06:59:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #18 0xa6022c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
06:59:29.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #19 0x7f306265fc1d in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3
06:59:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #20 0x7f30659e26da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
06:59:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     #21 0x7f305e82471e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
06:59:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46 in 
06:59:29.424 [DEBUG - kudu-nio-0] (Connection.java:239) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] handling channelInactive
06:59:29.424 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state READY due to: connection closed
06:59:29.424 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: connection closed
06:59:29.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:29.431 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:29.431 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=2, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=2, TimeoutTracker(timeout=30000, elapsed=1271), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2)))) after lookup
06:59:29.432 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:29.433 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.433 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.433 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.451 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:29.451 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:29.451 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=4, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=4, TimeoutTracker(timeout=30000, elapsed=1290), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244)))) after lookup
06:59:29.452 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:29.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.453 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.490 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:29.490 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:29.491 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=6, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=6, TimeoutTracker(timeout=30000, elapsed=1330), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891)))) after lookup
06:59:29.491 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:29.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.492 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.631 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:29.631 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299998, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:29.633 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=8, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=8, TimeoutTracker(timeout=30000, elapsed=1472), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684)))) after lookup
06:59:29.634 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:29.635 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.635 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.635 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.850 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:29.851 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:29.851 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=10, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=10, TimeoutTracker(timeout=30000, elapsed=1690), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401)))) after lookup
06:59:29.852 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:29.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:29.853 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:30.271 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:30.272 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:30.272 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=12, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=12, TimeoutTracker(timeout=30000, elapsed=2111), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625)))) after lookup
06:59:30.273 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:30.274 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:30.275 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:30.275 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:33.291 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:33.291 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:33.292 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=14, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=14, TimeoutTracker(timeout=30000, elapsed=5131), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633)))) after lookup
06:59:33.292 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:33.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:33.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:33.293 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.151 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:34.151 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:34.152 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=16, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=16, TimeoutTracker(timeout=30000, elapsed=5991), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845)))) after lookup
06:59:34.153 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:34.154 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.154 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.154 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.531 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:34.531 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:34.532 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=18, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=18, TimeoutTracker(timeout=30000, elapsed=6372), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789)))) after lookup
06:59:34.533 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:34.534 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.534 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:34.534 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:37.891 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:37.891 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:37.891 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=20, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=20, TimeoutTracker(timeout=30000, elapsed=9731), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825)))) after lookup
06:59:37.893 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:37.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:37.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:37.895 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:40.231 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:40.232 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:40.232 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=22, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=22, TimeoutTracker(timeout=30000, elapsed=12072), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631)))) after lookup
06:59:40.234 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:40.235 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:40.235 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:40.235 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:43.631 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:43.631 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:43.632 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=24, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=24, TimeoutTracker(timeout=30000, elapsed=15472), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983)))) after lookup
06:59:43.634 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:43.636 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:43.636 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:43.636 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.511 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:47.512 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:47.512 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=26, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=26, TimeoutTracker(timeout=30000, elapsed=19352), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746)))) after lookup
06:59:47.514 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:47.514 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.515 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.515 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.831 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:47.832 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:47.832 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=28, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=28, TimeoutTracker(timeout=30000, elapsed=19672), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024)))) after lookup
06:59:47.834 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:47.835 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.835 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:47.835 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:49.390 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:49.391 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:49.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=30, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=30, TimeoutTracker(timeout=30000, elapsed=21231), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943)))) after lookup
06:59:49.393 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:49.393 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:49.394 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:49.394 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:53.311 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:53.311 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:53.311 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=32, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=32, TimeoutTracker(timeout=30000, elapsed=25151), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085)))) after lookup
06:59:53.314 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:53.315 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:53.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:53.315 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:55.172 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:55.172 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:55.173 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=34, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=34, TimeoutTracker(timeout=30000, elapsed=27013), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853)))) after lookup
06:59:55.175 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:55.176 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:55.176 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:55.176 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:56.371 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:56.371 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:56.371 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=36, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=36, TimeoutTracker(timeout=30000, elapsed=28211), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483)))) after lookup
06:59:56.373 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:56.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:56.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:56.375 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:57.751 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:57.751 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:57.751 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=38, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=38, TimeoutTracker(timeout=30000, elapsed=29591), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276)))) after lookup
06:59:57.763 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:57.764 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:57.764 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:57.764 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:57.929 [DEBUG - pool-173-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=getTableKeyRanges, tablet=null, attempt=0, TimeoutTracker(timeout=30000, elapsed=30013), Traces: [0ms] sending RPC to server f6a9171e8c1d44838d03288acc734f59, [14ms] received response from server f6a9171e8c1d44838d03288acc734f59: OK, deferred=null) with no deferred attached!
06:59:57.930 [DEBUG - pool-173-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=getTableKeyRanges, tablet=null, attempt=0, TimeoutTracker(timeout=29994, elapsed=30008), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [8ms] received response from server eef349bbf8d24181926900a522fe9624: OK, deferred=null) with no deferred attached!
06:59:58.091 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:58.091 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:58.091 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=40, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=40, TimeoutTracker(timeout=30000, elapsed=29931), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753)))) after lookup
06:59:58.093 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:58.094 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.094 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.094 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.095 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
06:59:58.096 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

06:59:58.109 [ERROR - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:94) Exception in task 2.0 in stage 0.0 (TID 2)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160) ~[main/:?]
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

06:59:58.181 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 3.0 in stage 0.0 (TID 3) (dist-test-slave-7lpv.c.gcp-upstream.internal, executor driver, partition 3, ANY, 5328 bytes) taskResourceAssignments Map()
06:59:58.181 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (Logging.scala:57) Running task 3.0 in stage 0.0 (TID 3)
06:59:58.184 [WARN - task-result-getter-2] (Logging.scala:69) Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-7lpv.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

06:59:58.187 [ERROR - task-result-getter-2] (Logging.scala:73) Task 2 in stage 0.0 failed 1 times; aborting job
06:59:58.189 [DEBUG - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:58.190 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
06:59:58.190 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
06:59:58.192 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:58.193 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:58.193 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=null)) after lookup
06:59:58.193 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:58.194 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.194 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.195 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.197 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Executor is trying to kill task 3.0 in stage 0.0 (TID 3), reason: Stage cancelled
06:59:58.198 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
06:59:58.199 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (collectAsList at SparkSQLTest.scala:471) failed in 30.248 s due to Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-7lpv.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
06:59:58.202 [INFO - main] (Logging.scala:57) Job 0 failed: collectAsList at SparkSQLTest.scala:471, took 30.255791 s
06:59:58.212 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:58.212 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:58.213 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=3, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=3, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [3ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [4ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1525717609(state=PENDING, result=null, callback=(continuation of Deferred@175191660 after retry RPC@472105963), errback=(continuation of Deferred@175191660 after retry RPC@472105963)))) after lookup
06:59:58.213 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:58.215 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.215 [DEBUG - kudu-nio-0] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.215 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
06:59:58.218 [INFO - dispatcher-event-loop-0] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
06:59:58.222 [INFO - main] (Logging.scala:57) MemoryStore cleared
06:59:58.222 [INFO - main] (Logging.scala:57) BlockManager stopped
06:59:58.223 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
06:59:58.224 [INFO - dispatcher-event-loop-0] (Logging.scala:57) OutputCommitCoordinator stopped!
06:59:58.230 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
06:59:58.230 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet dbfe916efc16446abee11dc3c5524278 for table 'test' with partition [0x80000032, <end>)
06:59:58.231 [DEBUG - main] (Connection.java:575) [peer f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)] disconnecting while in state READY
06:59:58.231 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=dbfe916efc16446abee11dc3c5524278@[eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)[L]]}]
06:59:58.231 [DEBUG - main] (Connection.java:575) [peer master-127.19.41.254:38787(127.19.41.254:38787)] disconnecting while in state READY
06:59:58.231 [DEBUG - main] (Connection.java:575) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] disconnecting while in state TERMINATED
06:59:58.231 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=5, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=5, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [3ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [4ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [23ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [26ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [40ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, deferred=Deferred@1525717609(state=PENDING, result=null, callback=(continuation of Deferred@175191660 after retry RPC@472105963) -> (continuation of Deferred@747547704 after retry RPC@783290073), errback=(continuation of Deferred@175191660 after retry RPC@472105963) -> (continuation of Deferred@747547704 after retry RPC@783290073)))) after lookup
06:59:58.232 [DEBUG - main] (Connection.java:575) [peer master-127.19.41.252:38057(127.19.41.252:38057)] disconnecting while in state READY
06:59:58.232 [DEBUG - main] (Connection.java:575) [peer master-127.19.41.253:43055(127.19.41.253:43055)] disconnecting while in state READY
06:59:58.232 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.41.252:38057(127.19.41.252:38057)] handling channelInactive
06:59:58.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.41.252:38057(127.19.41.252:38057)] cleaning up while in state READY due to: connection closed
06:59:58.234 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.41.253:43055(127.19.41.253:43055)] handling channelInactive
06:59:58.234 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.41.253:43055(127.19.41.253:43055)] cleaning up while in state READY due to: connection closed
06:59:58.234 [DEBUG - kudu-nio-1] (Connection.java:748) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] connecting to peer
06:59:58.235 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:58.236 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 28 thread-local buffer(s) from thread: kudu-nio-0
06:59:58.236 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)] handling channelInactive
06:59:58.236 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f6a9171e8c1d44838d03288acc734f59(127.19.41.193:35953)] cleaning up while in state READY due to: connection closed
06:59:58.237 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.41.254:38787(127.19.41.254:38787)] handling channelInactive
06:59:58.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.41.254:38787(127.19.41.254:38787)] cleaning up while in state READY due to: connection closed
06:59:58.237 [INFO - kudu-nio-1] (Connection.java:425) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] lost connection to peer
06:59:58.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] cleaning up while in state CONNECTING due to: [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] lost connection to peer
06:59:58.238 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location eef349bbf8d24181926900a522fe9624(127.19.41.194:34805) for tablet dbfe916efc16446abee11dc3c5524278: [peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805)] lost connection to peer
06:59:58.238 [WARN - kudu-nio-1] (AsyncKuduClient.java:2800) Failed to schedule timer. Ignore this if we're shutting down.
java.lang.IllegalStateException: cannot be started once stopped
	at org.apache.kudu.shaded.io.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:372) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:447) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2795) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2343) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel.doClose(AbstractNioChannel.java:502) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doClose(NioSocketChannel.java:349) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:754) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.closeAll(NioEventLoop.java:838) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:595) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: cannot be started once stopped
	at org.apache.kudu.shaded.io.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:372)
	at org.apache.kudu.shaded.io.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:447)
	at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2795)
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2343)
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel.doClose(AbstractNioChannel.java:502)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doClose(NioSocketChannel.java:349)
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:754)
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)
	at org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.closeAll(NioEventLoop.java:838)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:595)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

06:59:58.240 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 40 thread-local buffer(s) from thread: kudu-nio-1
06:59:58.248 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.249069 19623 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu pid 19904
06:59:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.266577 20026 generic_service.cc:196] Checking for leaks (request via RPC)
06:59:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.400367 19623 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu with pid 19904
06:59:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.434549 19623 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu pid 20175
06:59:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.442324 20297 generic_service.cc:196] Checking for leaks (request via RPC)
06:59:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.551965 19623 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu with pid 20175
06:59:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.570392 19623 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu pid 19634
06:59:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.571669 19695 generic_service.cc:196] Checking for leaks (request via RPC)
06:59:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.685766 19623 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu with pid 19634
06:59:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.716101 19623 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu pid 19704
06:59:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.717229 19768 generic_service.cc:196] Checking for leaks (request via RPC)
06:59:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.829062 19623 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu with pid 19704
06:59:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.850736 19623 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu pid 19775
06:59:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.851969 19839 generic_service.cc:196] Checking for leaks (request via RPC)
06:59:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:58.986683 19623 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ9V9WF/build/asan/bin/kudu with pid 19775
06:59:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:59Z chronyd exiting
06:59:59.141 [ERROR - main] (RetryRule.java:206) org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange: failed attempt 1
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-7lpv.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-7lpv.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, KuduRpc(method=Scan, tablet=dbfe916efc16446abee11dc3c5524278, attempt=41, TimeoutTracker(timeout=30000, elapsed=29934), Traces: [0ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1264ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: connection closed, [1264ms] delaying RPC due to: Network error: connection closed, [1268ms] refreshing cache from master, [1269ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1270ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1272ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1273ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1273ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1289ms] refreshing cache from master, [1289ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1290ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1291ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1293ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1293ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1328ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1330ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1331ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1332ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1333ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1469ms] refreshing cache from master, [1469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1473ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1475ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1688ms] refreshing cache from master, [1688ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [1690ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [1692ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [1693ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [1693ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2109ms] refreshing cache from master, [2110ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [2111ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [2112ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [2115ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [2115ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5129ms] refreshing cache from master, [5129ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5131ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5132ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5133ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5133ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5989ms] refreshing cache from master, [5989ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [5991ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [5993ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [5994ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [5995ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6369ms] refreshing cache from master, [6369ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [6371ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [6373ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [6374ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [6374ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9729ms] refreshing cache from master, [9729ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [9731ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [9733ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [9735ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [9735ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12069ms] refreshing cache from master, [12069ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [12071ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [12074ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [12075ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [12075ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15469ms] refreshing cache from master, [15469ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [15471ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [15474ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [15476ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [15476ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19349ms] refreshing cache from master, [19349ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19351ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19353ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19355ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19355ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19669ms] refreshing cache from master, [19669ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [19671ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [19674ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [19675ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [19675ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21228ms] refreshing cache from master, [21228ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [21230ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [21232ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [21234ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [21234ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25149ms] refreshing cache from master, [25149ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.41.254:38787, [25150ms] Sub RPC GetTableLocations: received response from server master-127.19.41.254:38787: OK, [25153ms] sending RPC to server eef349bbf8d24181926900a522fe9624, [25155ms] received response from server eef349bbf8d24181926900a522fe9624: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [25155ms] delaying RPC due to: Network error: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805, [27009ms] refreshing cache from master, [27009ms] too many traces: truncated at 100 traces, deferred=Deferred@1129654812(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@1098257088 after retry RPC@1716328244) -> (continuation of Deferred@53147549 after retry RPC@223392891) -> (continuation of Deferred@399327905 after retry RPC@94405684) -> (continuation of Deferred@750543350 after retry RPC@758956401) -> (continuation of Deferred@1546717263 after retry RPC@1079971625) -> (continuation of Deferred@1010448136 after retry RPC@77873633) -> (continuation of Deferred@1084315428 after retry RPC@1431664845) -> (continuation of Deferred@1443796241 after retry RPC@677579789) -> (continuation of Deferred@704813592 after retry RPC@812250825) -> (continuation of Deferred@1521776747 after retry RPC@1022903631) -> (continuation of Deferred@1516708600 after retry RPC@82672983) -> (continuation of Deferred@2125453989 after retry RPC@289867746) -> (continuation of Deferred@109252666 after retry RPC@1163802024) -> (continuation of Deferred@2038611330 after retry RPC@1296376943) -> (continuation of Deferred@620340108 after retry RPC@1013851085) -> (continuation of Deferred@540607300 after retry RPC@1837382853) -> (continuation of Deferred@1395462606 after retry RPC@2136671483) -> (continuation of Deferred@2065116769 after retry RPC@1556906276) -> (continuation of Deferred@1752857870 after retry RPC@37950753) -> (continuation of Deferred@1101062062 after retry RPC@2008449031))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 19 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer eef349bbf8d24181926900a522fe9624(127.19.41.194:34805): Connection refused: /127.19.41.194:34805
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.41.194:34805
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)
	at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
	at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
	at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696)
	at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977)
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)
	at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976)
	at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
	at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)