Unable to diagnose

06:59:49.130 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:59:49.130 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasklQverR/test-tmp
06:59:49.130 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
06:59:49.130 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasklQverR/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
06:59:49.132 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807"
  extra_tserver_flags: "--flush_threshold_mb=1"
  extra_tserver_flags: "--flush_threshold_secs=1"
  extra_tserver_flags: "--enable_rowset_compaction=false"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

06:59:49.207 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:59:49.207 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

06:59:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.207800 10618 mini_oidc.cc:152] Starting JWKS server
06:59:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.208505 10618 webserver.cc:460] Webserver started at http://127.0.0.1:37479/ using document root <none> and password file <none>
06:59:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.208580 10618 mini_oidc.cc:189] Starting OIDC Discovery server
06:59:49.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.208736 10618 webserver.cc:460] Webserver started at http://127.0.0.1:41847/ using document root <none> and password file <none>
06:59:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:59:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:49Z Disabled control of system clock
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.219467 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/logs
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data/info.pb
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.190:37303
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.190
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json with env {}
06:59:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.298030 10629 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.298210 10629 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.298242 10629 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299638 10629 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299695 10629 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299716 10629 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299734 10629 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299749 10629 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.299764 10629 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37479/jwks.json
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.301206 10629 master_runner.cc:386] Master server non-default flags:
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.190:37303
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data/info.pb
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.190
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.10629
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/logs
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.301425 10629 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.301672 10629 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.306217 10634 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.306632 10637 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.307212 10635 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.307879 10629 server_base.cc:1034] running on GCE node
06:59:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.308048 10629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.308230 10629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:49.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.309370 10629 hybrid_clock.cc:648] HybridClock initialized: now 1714114789309354 us; error 36 us; skew 500 ppm
06:59:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.314402 10629 webserver.cc:460] Webserver started at http://127.10.94.190:44423/ using document root <none> and password file <none>
06:59:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.314594 10629 fs_manager.cc:359] Metadata directory not provided
06:59:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.314635 10629 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:49.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.314713 10629 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.315511 10629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data/instance:
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c18b20b95cad4b4a8ede1f4f887d0f72"
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.315769 10629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal/instance:
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c18b20b95cad4b4a8ede1f4f887d0f72"
06:59:49.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.316968 10629 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.317543 10643 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.317669 10629 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.317730 10629 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c18b20b95cad4b4a8ede1f4f887d0f72"
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.317775 10629 fs_report.cc:383] FS layout report
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data/data
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:49.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.336369 10629 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.336601 10629 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.336683 10629 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.339942 10629 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.190:37303
06:59:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.339969 10695 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.190:37303 every 8 connection(s)
06:59:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.340214 10629 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/data/info.pb
06:59:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.340782 10696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.342252 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.343410 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 10629
06:59:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.343520 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-0/wal/instance
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.344393 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/logs
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data/info.pb
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.189:37271
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.189
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json with env {}
06:59:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.348104 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.348461 10645 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.94.189:37271: connect: Connection refused (error 111)
06:59:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.348837 10696 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.189:37271: Network error: Client connection negotiation failed: client connection to 127.10.94.189:37271: connect: Connection refused (error 111)
06:59:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.398032 10696 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } attempt: 1
06:59:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.398770 10696 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.189:37271: Network error: Client connection negotiation failed: client connection to 127.10.94.189:37271: connect: Connection refused (error 111)
06:59:49.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.418321 10700 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:49.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.418479 10700 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:49.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.418511 10700 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419734 10700 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419775 10700 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419796 10700 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419813 10700 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:49.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419829 10700 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:49.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.419848 10700 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37479/jwks.json
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.421268 10700 master_runner.cc:386] Master server non-default flags:
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.189:37271
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data/info.pb
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.189
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.10700
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/logs
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.421464 10700 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.421700 10700 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.426117 10706 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.426620 10707 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.427053 10709 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.427791 10700 server_base.cc:1034] running on GCE node
06:59:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.427963 10700 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.428140 10700 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.429250 10700 hybrid_clock.cc:648] HybridClock initialized: now 1714114789429232 us; error 28 us; skew 500 ppm
06:59:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.430399 10700 webserver.cc:460] Webserver started at http://127.10.94.189:46435/ using document root <none> and password file <none>
06:59:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.430574 10700 fs_manager.cc:359] Metadata directory not provided
06:59:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.430615 10700 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.430689 10700 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.431533 10700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data/instance:
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7602761493674efcb4c167e3abd06416"
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.431788 10700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal/instance:
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7602761493674efcb4c167e3abd06416"
06:59:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.432799 10700 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.002s
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.433312 10715 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.433473 10700 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.433555 10700 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7602761493674efcb4c167e3abd06416"
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.433611 10700 fs_report.cc:383] FS layout report
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data/data
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.442481 10700 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.442723 10700 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.442835 10700 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.446249 10700 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.189:37271
06:59:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.446285 10767 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.189:37271 every 8 connection(s)
06:59:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.446604 10700 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/data/info.pb
06:59:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.447156 10768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.448575 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 10700
06:59:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.448642 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-1/wal/instance
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.448577 10768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.448895 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/logs
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data/info.pb
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.188:34325
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.188
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json with env {}
06:59:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.452929 10768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.456239 10768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.456629 10719 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.457126 10768 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.466939 10696 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } attempt: 2
06:59:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.470453 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.470983 10696 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.506373 10768 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 1
06:59:49.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.507103 10768 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.514145 10696 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 1
06:59:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.514905 10696 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.527189 10771 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.527352 10771 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.527370 10771 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528512 10771 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528544 10771 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528556 10771 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528568 10771 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528578 10771 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.528589 10771 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37479/jwks.json
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.529814 10771 master_runner.cc:386] Master server non-default flags:
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal
06:59:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.188:34325
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37479/jwks.json
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data/info.pb
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.188
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.10771
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/logs
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.529964 10771 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.530122 10771 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.534162 10778 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.534628 10779 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.535111 10781 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.535775 10771 server_base.cc:1034] running on GCE node
06:59:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.535959 10771 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.536155 10771 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:49.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.537288 10771 hybrid_clock.cc:648] HybridClock initialized: now 1714114789537271 us; error 33 us; skew 500 ppm
06:59:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.538431 10771 webserver.cc:460] Webserver started at http://127.10.94.188:46007/ using document root <none> and password file <none>
06:59:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.538641 10771 fs_manager.cc:359] Metadata directory not provided
06:59:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.538681 10771 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.538751 10771 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.539523 10771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data/instance:
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24afb7c4dd44773937c4c942642b13a"
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.539773 10771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal/instance:
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24afb7c4dd44773937c4c942642b13a"
06:59:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.540753 10771 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.002s
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.541298 10787 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.541472 10771 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.541553 10771 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24afb7c4dd44773937c4c942642b13a"
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:49 on dist-test-slave-str9"
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.541618 10771 fs_report.cc:383] FS layout report
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data/data
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.562076 10696 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 2
06:59:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.571367 10696 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.572542 10771 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.572818 10771 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.572926 10771 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.575307 10768 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 2
06:59:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.576071 10768 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.10.94.188:34325: Network error: Client connection negotiation failed: client connection to 127.10.94.188:34325: connect: Connection refused (error 111)
06:59:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.576916 10771 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.188:34325
06:59:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.576961 10839 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.188:34325 every 8 connection(s)
06:59:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.577299 10771 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/data/info.pb
06:59:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.577965 10840 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.580241 10840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.581210 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 10771
06:59:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.581279 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/master-2/wal/instance
06:59:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.585418 10840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.589521 10840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } has no permanent_uuid. Determining permanent_uuid...
06:59:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.594153 10840 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: Bootstrap starting.
06:59:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.594691 10840 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: Neither blocks nor log segments found. Creating new log.
06:59:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.594902 10840 log.cc:824] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: Log is configured to *not* fsync() on all Append() calls
06:59:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.595440 10840 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: No bootstrap required, opened a new log
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.596828 10840 raft_consensus.cc:348] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.596938 10840 raft_consensus.cc:374] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.596956 10840 raft_consensus.cc:729] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24afb7c4dd44773937c4c942642b13a, State: Initialized, Role: FOLLOWER
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.597028 10840 consensus_queue.cc:260] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.597301 10840 sys_catalog.cc:564] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: configured and running, proceeding with master startup.
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.597304 10849 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.597410 10849 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.598874 10860 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.598927 10860 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.599174 10771 master_runner.cc:186] Error getting master registration for 127.10.94.190:37303: OK, instance_id { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" instance_seqno: 1714114789338969 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.600793 10771 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:49.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.667258 10768 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 3
06:59:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.671278 10768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: Bootstrap starting.
06:59:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.671999 10768 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: Neither blocks nor log segments found. Creating new log.
06:59:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.672493 10768 log.cc:824] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: Log is configured to *not* fsync() on all Append() calls
06:59:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.673286 10768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: No bootstrap required, opened a new log
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.674980 10768 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675115 10768 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675138 10768 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7602761493674efcb4c167e3abd06416, State: Initialized, Role: FOLLOWER
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675225 10768 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675511 10768 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: configured and running, proceeding with master startup.
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675504 10863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.675598 10863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.677379 10874 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.677536 10874 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.678581 10696 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } attempt: 3
06:59:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.677546 10700 master_runner.cc:186] Error getting master registration for 127.10.94.190:37303: OK, instance_id { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" instance_seqno: 1714114789338969 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.679158 10700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.681960 10696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: Bootstrap starting.
06:59:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.682484 10696 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: Neither blocks nor log segments found. Creating new log.
06:59:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.683990 10696 log.cc:824] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: Log is configured to *not* fsync() on all Append() calls
06:59:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.684556 10696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: No bootstrap required, opened a new log
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.685830 10696 raft_consensus.cc:348] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.685918 10696 raft_consensus.cc:374] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.685947 10696 raft_consensus.cc:729] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c18b20b95cad4b4a8ede1f4f887d0f72, State: Initialized, Role: FOLLOWER
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.686092 10696 consensus_queue.cc:260] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.686506 10877 sys_catalog.cc:455] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.686707 10877 sys_catalog.cc:458] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.687116 10696 sys_catalog.cc:564] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: configured and running, proceeding with master startup.
06:59:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.688731 10888 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.688880 10888 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.690441 10629 master_runner.cc:263] No leader master found from master c18b20b95cad4b4a8ede1f4f887d0f72
06:59:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.690481 10629 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.693414 10721 tablet.cc:2369] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.823555 10793 tablet.cc:2369] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:49.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.837278 10649 tablet.cc:2369] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:49.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896111 10849 raft_consensus.cc:482] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:49.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896257 10849 raft_consensus.cc:504] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896534 10849 leader_election.cc:288] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c18b20b95cad4b4a8ede1f4f887d0f72 (127.10.94.190:37303), 7602761493674efcb4c167e3abd06416 (127.10.94.189:37271)
06:59:49.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896800 10742 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24afb7c4dd44773937c4c942642b13a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7602761493674efcb4c167e3abd06416" is_pre_election: true
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896845 10670 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24afb7c4dd44773937c4c942642b13a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" is_pre_election: true
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896932 10742 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24afb7c4dd44773937c4c942642b13a in term 0.
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.896977 10670 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24afb7c4dd44773937c4c942642b13a in term 0.
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.897168 10791 leader_election.cc:302] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a24afb7c4dd44773937c4c942642b13a, c18b20b95cad4b4a8ede1f4f887d0f72; no voters: 
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.897361 10849 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Leader pre-election won for term 1
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.897428 10849 raft_consensus.cc:482] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:59:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.897452 10849 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 0 FOLLOWER]: Advancing to term 1
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898137 10849 raft_consensus.cc:504] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898260 10849 leader_election.cc:288] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [CANDIDATE]: Term 1 election: Requested vote from peers c18b20b95cad4b4a8ede1f4f887d0f72 (127.10.94.190:37303), 7602761493674efcb4c167e3abd06416 (127.10.94.189:37271)
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898412 10742 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24afb7c4dd44773937c4c942642b13a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7602761493674efcb4c167e3abd06416"
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898468 10670 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24afb7c4dd44773937c4c942642b13a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72"
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898491 10742 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 0 FOLLOWER]: Advancing to term 1
06:59:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.898559 10670 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 0 FOLLOWER]: Advancing to term 1
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899168 10670 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24afb7c4dd44773937c4c942642b13a in term 1.
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899205 10742 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24afb7c4dd44773937c4c942642b13a in term 1.
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899366 10791 leader_election.cc:302] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a24afb7c4dd44773937c4c942642b13a, c18b20b95cad4b4a8ede1f4f887d0f72; no voters: 
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899470 10849 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 1 FOLLOWER]: Leader election won for term 1
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899606 10849 raft_consensus.cc:686] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [term 1 LEADER]: Becoming Leader. State: Replica: a24afb7c4dd44773937c4c942642b13a, State: Running, Role: LEADER
06:59:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.899681 10849 consensus_queue.cc:237] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } }
06:59:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.900041 10890 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24afb7c4dd44773937c4c942642b13a. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.900120 10890 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: This master's current role is: LEADER
06:59:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.900318 10892 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:59:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.900488 10892 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:59:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.901629 10670 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [term 1 FOLLOWER]: Refusing update from remote peer a24afb7c4dd44773937c4c942642b13a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.901660 10742 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [term 1 FOLLOWER]: Refusing update from remote peer a24afb7c4dd44773937c4c942642b13a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.901909 10890 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.901997 10849 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903059 10863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24afb7c4dd44773937c4c942642b13a. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903144 10863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903199 10877 sys_catalog.cc:455] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24afb7c4dd44773937c4c942642b13a. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903264 10877 sys_catalog.cc:458] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903816 10849 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.903901 10849 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: This master's current role is: LEADER
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904148 10877 sys_catalog.cc:455] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904206 10877 sys_catalog.cc:458] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904247 10900 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904315 10900 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a [sys.catalog]: This master's current role is: LEADER
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904615 10863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a24afb7c4dd44773937c4c942642b13a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c18b20b95cad4b4a8ede1f4f887d0f72" member_type: VOTER last_known_addr { host: "127.10.94.190" port: 37303 } } peers { permanent_uuid: "7602761493674efcb4c167e3abd06416" member_type: VOTER last_known_addr { host: "127.10.94.189" port: 37271 } } peers { permanent_uuid: "a24afb7c4dd44773937c4c942642b13a" member_type: VOTER last_known_addr { host: "127.10.94.188" port: 34325 } } }
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904635 10895 mvcc.cc:205] Tried to move back new op lower bound from 7021014179435061248 to 7021014179429564416. Current Snapshot: MvccSnapshot[applied={T|T < 7021014179435061248}]
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904680 10863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416 [sys.catalog]: This master's current role is: FOLLOWER
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904737 10892 catalog_manager.cc:1347] Generated new cluster ID: bf8fc0df8d204989893c99206ed3b73e
06:59:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.904778 10892 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:59:49.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.911159 10892 catalog_manager.cc:1370] Generated new certificate authority record
06:59:49.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.911794 10892 catalog_manager.cc:1504] Loading token signing keys...
06:59:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.915879 10892 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a24afb7c4dd44773937c4c942642b13a: Generated new TSK 0
06:59:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.916064 10892 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:59:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:49.928849 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/logs
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data/info.pb
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.129:0
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.129
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.129
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.003150 10902 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.003306 10902 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.003324 10902 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.003337 10902 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.004549 10902 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.004617 10902 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.004629 10902 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.004644 10902 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.94.129
06:59:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.005899 10902 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.129:0
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data/info.pb
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.129
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.10902
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.129
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/logs
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.006084 10902 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.006273 10902 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:50.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.010577 10907 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.010864 10908 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.011318 10910 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.012068 10902 server_base.cc:1034] running on GCE node
06:59:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.012193 10902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.012351 10902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.013470 10902 hybrid_clock.cc:648] HybridClock initialized: now 1714114790013459 us; error 24 us; skew 500 ppm
06:59:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.014593 10902 webserver.cc:460] Webserver started at http://127.10.94.129:37745/ using document root <none> and password file <none>
06:59:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.014768 10902 fs_manager.cc:359] Metadata directory not provided
06:59:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.014811 10902 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.014909 10902 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.015724 10902 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data/instance:
06:59:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0154f5115a5d4ca9b2e2cced3e3eb545"
06:59:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.016016 10902 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal/instance:
06:59:50.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0154f5115a5d4ca9b2e2cced3e3eb545"
06:59:50.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.017196 10902 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.002s
06:59:50.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.017838 10916 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.018011 10902 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.018085 10902 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0154f5115a5d4ca9b2e2cced3e3eb545"
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.018147 10902 fs_report.cc:383] FS layout report
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data/data
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.030414 10902 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.030633 10902 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.030731 10902 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.030910 10902 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.031175 10902 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.031208 10902 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.031236 10902 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.031252 10902 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.036403 10902 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.129:36247
06:59:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.036461 11029 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.129:36247 every 8 connection(s)
06:59:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.036729 10902 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/data/info.pb
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041510 11030 heartbeater.cc:347] Connected to a master server at 127.10.94.188:34325
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041518 11032 heartbeater.cc:347] Connected to a master server at 127.10.94.189:37271
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041622 11030 heartbeater.cc:464] Registering TS with master...
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041622 11032 heartbeater.cc:464] Registering TS with master...
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041810 11032 heartbeater.cc:511] Master 127.10.94.189:37271 requested a full tablet report, sending...
06:59:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.041810 11030 heartbeater.cc:511] Master 127.10.94.188:34325 requested a full tablet report, sending...
06:59:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042078 11031 heartbeater.cc:347] Connected to a master server at 127.10.94.190:37303
06:59:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042233 11031 heartbeater.cc:464] Registering TS with master...
06:59:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042246 10804 ts_manager.cc:194] Registered new tserver with Master: 0154f5115a5d4ca9b2e2cced3e3eb545 (127.10.94.129:36247)
06:59:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042279 10732 ts_manager.cc:194] Registered new tserver with Master: 0154f5115a5d4ca9b2e2cced3e3eb545 (127.10.94.129:36247)
06:59:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042815 11031 heartbeater.cc:511] Master 127.10.94.190:37303 requested a full tablet report, sending...
06:59:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042918 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 10902
06:59:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.042997 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-0/wal/instance
06:59:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.043141 10804 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.94.129:44041
06:59:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.043248 10660 ts_manager.cc:194] Registered new tserver with Master: 0154f5115a5d4ca9b2e2cced3e3eb545 (127.10.94.129:36247)
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.044209 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/logs
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data/info.pb
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.130:0
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.130
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.130
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:50.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.123384 11037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.123534 11037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.123551 11037 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.123565 11037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.124687 11037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.124723 11037 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.124734 11037 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.124749 11037 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.94.130
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.126034 11037 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.130:0
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data/info.pb
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.130
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.11037
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.130
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/logs
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.126201 11037 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.126379 11037 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.130906 11043 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.131405 11042 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.131942 11045 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.132460 11037 server_base.cc:1034] running on GCE node
06:59:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.132578 11037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.132723 11037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.133858 11037 hybrid_clock.cc:648] HybridClock initialized: now 1714114790133842 us; error 34 us; skew 500 ppm
06:59:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.134963 11037 webserver.cc:460] Webserver started at http://127.10.94.130:43023/ using document root <none> and password file <none>
06:59:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.135130 11037 fs_manager.cc:359] Metadata directory not provided
06:59:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.135162 11037 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.135231 11037 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.136035 11037 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data/instance:
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686537b603c94bc3a797ce5808b5d5bf"
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.136344 11037 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal/instance:
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686537b603c94bc3a797ce5808b5d5bf"
06:59:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.137425 11037 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.138002 11051 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.138166 11037 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.138243 11037 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686537b603c94bc3a797ce5808b5d5bf"
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.138305 11037 fs_report.cc:383] FS layout report
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data/data
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.153440 11037 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.153673 11037 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.153784 11037 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.153998 11037 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.154296 11037 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.154328 11037 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.154358 11037 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.154376 11037 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.159466 11037 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.130:33471
06:59:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.159500 11164 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.130:33471 every 8 connection(s)
06:59:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.159773 11037 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/data/info.pb
06:59:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164331 11166 heartbeater.cc:347] Connected to a master server at 127.10.94.190:37303
06:59:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164547 11166 heartbeater.cc:464] Registering TS with master...
06:59:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164628 11167 heartbeater.cc:347] Connected to a master server at 127.10.94.189:37271
06:59:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164695 11167 heartbeater.cc:464] Registering TS with master...
06:59:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164837 11167 heartbeater.cc:511] Master 127.10.94.189:37271 requested a full tablet report, sending...
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164881 11165 heartbeater.cc:347] Connected to a master server at 127.10.94.188:34325
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164849 11166 heartbeater.cc:511] Master 127.10.94.190:37303 requested a full tablet report, sending...
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.164937 11165 heartbeater.cc:464] Registering TS with master...
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.165014 11165 heartbeater.cc:511] Master 127.10.94.188:34325 requested a full tablet report, sending...
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.165313 10804 ts_manager.cc:194] Registered new tserver with Master: 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471)
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.165323 10660 ts_manager.cc:194] Registered new tserver with Master: 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471)
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.165323 10732 ts_manager.cc:194] Registered new tserver with Master: 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471)
06:59:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.165721 10804 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.94.130:43267
06:59:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.168463 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 11037
06:59:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.168520 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-1/wal/instance
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.169425 10618 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasklQverR/build/release/bin/kudu
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/logs
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data/info.pb
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.131:0
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.131
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.131
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:59:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.244010 11172 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.244171 11172 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.244199 11172 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:59:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.244225 11172 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:50.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.245590 11172 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:50.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.245642 11172 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:59:50.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.245663 11172 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:59:50.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.245688 11172 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.94.131
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.247113 11172 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.94.148:36941
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.94.131:0
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data/info.pb
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.94.131
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.94.190:37303,127.10.94.189:37271,127.10.94.188:34325
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.11172
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.94.131
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/logs
06:59:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.247329 11172 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.247584 11172 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.252027 11177 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.252518 11180 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.253006 11178 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.253710 11172 server_base.cc:1034] running on GCE node
06:59:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.253893 11172 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.254109 11172 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.255247 11172 hybrid_clock.cc:648] HybridClock initialized: now 1714114790255224 us; error 34 us; skew 500 ppm
06:59:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.256383 11172 webserver.cc:460] Webserver started at http://127.10.94.131:39105/ using document root <none> and password file <none>
06:59:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.256570 11172 fs_manager.cc:359] Metadata directory not provided
06:59:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.256619 11172 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.256727 11172 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.257583 11172 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data/instance:
06:59:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdde07f1fbaf496eb82c215b5e7a0215"
06:59:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.257899 11172 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal/instance:
06:59:50.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdde07f1fbaf496eb82c215b5e7a0215"
06:59:50.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.259016 11172 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
06:59:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.259575 11186 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:59:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.259771 11172 fs_manager.cc:725] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
06:59:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.259850 11172 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data,/tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal
06:59:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdde07f1fbaf496eb82c215b5e7a0215"
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:50 on dist-test-slave-str9"
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.259929 11172 fs_report.cc:383] FS layout report
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data/data
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:50.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.270776 11172 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.270988 11172 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271087 11172 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271255 11172 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271519 11172 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271549 11172 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271582 11172 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.271597 11172 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:59:50.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.276744 11172 rpc_server.cc:307] RPC server started. Bound to: 127.10.94.131:46695
06:59:50.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.276770 11299 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.94.131:46695 every 8 connection(s)
06:59:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.277081 11172 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/data/info.pb
06:59:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.281605 11302 heartbeater.cc:347] Connected to a master server at 127.10.94.189:37271
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.281702 11302 heartbeater.cc:464] Registering TS with master...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.281841 11302 heartbeater.cc:511] Master 127.10.94.189:37271 requested a full tablet report, sending...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282229 10732 ts_manager.cc:194] Registered new tserver with Master: cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695)
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282387 11301 heartbeater.cc:347] Connected to a master server at 127.10.94.190:37303
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282440 11301 heartbeater.cc:464] Registering TS with master...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282526 11301 heartbeater.cc:511] Master 127.10.94.190:37303 requested a full tablet report, sending...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282569 11300 heartbeater.cc:347] Connected to a master server at 127.10.94.188:34325
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282616 11300 heartbeater.cc:464] Registering TS with master...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282701 11300 heartbeater.cc:511] Master 127.10.94.188:34325 requested a full tablet report, sending...
06:59:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282765 10660 ts_manager.cc:194] Registered new tserver with Master: cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695)
06:59:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.282943 10804 ts_manager.cc:194] Registered new tserver with Master: cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695)
06:59:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.283252 10804 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.94.131:44471
06:59:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.283299 10618 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasklQverR/build/release/bin/kudu as pid 11172
06:59:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.283383 10618 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasklQverR/test-tmp/mini-kudu-cluster6157751816386495807/ts-2/wal/instance
06:59:50.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.285648 10618 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:59:50.286 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:59:50.286 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

06:59:50.286 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.10.94.190"
      port: 37303
    }
    bound_http_address {
      host: "127.10.94.190"
      port: 44423
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.10.94.189"
      port: 37271
    }
    bound_http_address {
      host: "127.10.94.189"
      port: 46435
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.10.94.188"
      port: 34325
    }
    bound_http_address {
      host: "127.10.94.188"
      port: 46007
    }
  }
}

06:59:50.287 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

06:59:50.287 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.10.94.129"
      port: 36247
    }
    bound_http_address {
      host: "127.10.94.129"
      port: 37745
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.10.94.130"
      port: 33471
    }
    bound_http_address {
      host: "127.10.94.130"
      port: 43023
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.10.94.131"
      port: 46695
    }
    bound_http_address {
      host: "127.10.94.131"
      port: 39105
    }
  }
}

06:59:50.287 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:59:50.289 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:59:50.289 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
06:59:50.290 [INFO - main] (Logging.scala:57) ==============================================================
06:59:50.290 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
06:59:50.290 [INFO - main] (Logging.scala:57) ==============================================================
06:59:50.290 [INFO - main] (Logging.scala:57) Submitted application: test
06:59:50.290 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
06:59:50.290 [INFO - main] (Logging.scala:57) Limiting resource is cpu
06:59:50.290 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
06:59:50.291 [INFO - main] (Logging.scala:57) Changing view acls to: slave
06:59:50.291 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
06:59:50.291 [INFO - main] (Logging.scala:57) Changing view acls groups to: 
06:59:50.291 [INFO - main] (Logging.scala:57) Changing modify acls groups to: 
06:59:50.291 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(slave); groups with view permissions: Set(); users  with modify permissions: Set(slave); groups with modify permissions: Set()
06:59:50.308 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 45095.
06:59:50.309 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
06:59:50.310 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
06:59:50.310 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
06:59:50.310 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
06:59:50.311 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
06:59:50.311 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-22999522-bd7f-4a85-80aa-7df64d50693c
06:59:50.311 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
06:59:50.312 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
06:59:50.328 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-str9.c.gcp-upstream.internal
06:59:50.331 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45795.
06:59:50.331 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-str9.c.gcp-upstream.internal:45795
06:59:50.331 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
06:59:50.332 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-str9.c.gcp-upstream.internal, 45795, None)
06:59:50.332 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-str9.c.gcp-upstream.internal:45795 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-str9.c.gcp-upstream.internal, 45795, None)
06:59:50.333 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-str9.c.gcp-upstream.internal, 45795, None)
06:59:50.333 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-str9.c.gcp-upstream.internal, 45795, None)
06:59:50.339 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:59:50.350 [DEBUG - main] (Connection.java:748) [peer master-127.10.94.190:37303(127.10.94.190:37303)] connecting to peer
06:59:50.351 [DEBUG - main] (Connection.java:748) [peer master-127.10.94.188:34325(127.10.94.188:34325)] connecting to peer
06:59:50.351 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.94.190:37303(127.10.94.190:37303)] Successfully connected to peer
06:59:50.351 [DEBUG - main] (Connection.java:748) [peer master-127.10.94.189:37271(127.10.94.189:37271)] connecting to peer
06:59:50.352 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.94.188:34325(127.10.94.188:34325)] Successfully connected to peer
06:59:50.353 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.94.190
06:59:50.354 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.94.188
06:59:50.357 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.94.189:37271(127.10.94.189:37271)] Successfully connected to peer
06:59:50.362 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:50.362 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:50.364 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.94.189
06:59:50.364 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2c04e99e, L:/127.0.0.1:35364 - R:/127.10.94.188:34325] using SASL/PLAIN
06:59:50.365 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:59:50.367 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6603b907, L:/127.0.0.1:59734 - R:/127.10.94.190:37303] using SASL/PLAIN
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.366652 10804 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35364:
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c1_i"
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c2_s"
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c3_double"
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DOUBLE
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c4_long"
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c5_bool"
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c6_short"
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT16
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c7_float"
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: FLOAT
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c8_binary"
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BINARY
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c9_unixtime_micros"
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: UNIXTIME_MICROS
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c10_byte"
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c11_decimal32"
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL32
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 9
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c12_decimal64"
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL64
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 18
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c13_decimal128"
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL128
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 38
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c14_varchar"
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: VARCHAR
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       length: 65535
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c15_date"
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DATE
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:59:50.372 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
06:59:50.374 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 209d7348864c4848a26b67ceb94aead1
06:59:50.374 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6848198a, L:/127.0.0.1:36938 - R:/127.10.94.189:37271] using SASL/PLAIN
06:59:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.377893 11099 tablet_service.cc:1467] Processing CreateTablet for tablet 9a6bad4b19d742f5810c6f7f0d60cdfe (DEFAULT_TABLE table=test [id=209d7348864c4848a26b67ceb94aead1]), partition=RANGE (key) PARTITION 50 <= VALUES
06:59:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.378230 11099 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9a6bad4b19d742f5810c6f7f0d60cdfe. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.378515 11234 tablet_service.cc:1467] Processing CreateTablet for tablet a026f316ccae47cf8497f46a1633f13b (DEFAULT_TABLE table=test [id=209d7348864c4848a26b67ceb94aead1]), partition=RANGE (key) PARTITION VALUES < 50
06:59:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.378827 11234 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a026f316ccae47cf8497f46a1633f13b. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.380810 11342 tablet_bootstrap.cc:492] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: Bootstrap starting.
06:59:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.381430 11342 tablet_bootstrap.cc:654] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: Neither blocks nor log segments found. Creating new log.
06:59:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.381901 11342 log.cc:824] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: Log is configured to *not* fsync() on all Append() calls
06:59:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.382552 11342 tablet_bootstrap.cc:492] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: No bootstrap required, opened a new log
06:59:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.382619 11342 ts_tablet_manager.cc:1367] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: Time spent bootstrapping tablet: real 0.002s	user 0.001s	sys 0.000s
06:59:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.382691 11343 tablet_bootstrap.cc:492] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: Bootstrap starting.
06:59:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.383558 11343 tablet_bootstrap.cc:654] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: Neither blocks nor log segments found. Creating new log.
06:59:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.383855 11343 log.cc:824] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: Log is configured to *not* fsync() on all Append() calls
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384526 11343 tablet_bootstrap.cc:492] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: No bootstrap required, opened a new log
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384596 11343 ts_tablet_manager.cc:1367] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: Time spent bootstrapping tablet: real 0.002s	user 0.001s	sys 0.000s
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384534 11342 raft_consensus.cc:348] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384646 11342 raft_consensus.cc:374] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384676 11342 raft_consensus.cc:729] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdde07f1fbaf496eb82c215b5e7a0215, State: Initialized, Role: FOLLOWER
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384780 11342 consensus_queue.cc:260] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384855 11342 raft_consensus.cc:388] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384886 11342 raft_consensus.cc:482] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.384922 11342 raft_consensus.cc:3037] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Advancing to term 1
06:59:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.385557 11342 raft_consensus.cc:504] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.385661 11342 leader_election.cc:302] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: cdde07f1fbaf496eb82c215b5e7a0215; no voters: 
06:59:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.385824 11342 leader_election.cc:288] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.385879 11346 raft_consensus.cc:2781] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 1 FOLLOWER]: Leader election won for term 1
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386149 11342 ts_tablet_manager.cc:1398] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215: Time spent starting tablet: real 0.003s	user 0.003s	sys 0.000s
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386204 11346 raft_consensus.cc:686] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [term 1 LEADER]: Becoming Leader. State: Replica: cdde07f1fbaf496eb82c215b5e7a0215, State: Running, Role: LEADER
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386366 11346 consensus_queue.cc:237] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386440 11343 raft_consensus.cc:348] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } }
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386523 11343 raft_consensus.cc:374] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386577 11343 raft_consensus.cc:729] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 686537b603c94bc3a797ce5808b5d5bf, State: Initialized, Role: FOLLOWER
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386673 11343 consensus_queue.cc:260] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } }
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386730 11343 raft_consensus.cc:388] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386749 11343 raft_consensus.cc:482] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.386775 11343 raft_consensus.cc:3037] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Advancing to term 1
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387037 11300 heartbeater.cc:503] Master 127.10.94.188:34325 was elected leader, sending a full tablet report...
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387337 11343 raft_consensus.cc:504] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } }
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387429 11343 leader_election.cc:302] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 686537b603c94bc3a797ce5808b5d5bf; no voters: 
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387562 11343 leader_election.cc:288] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387727 11343 ts_tablet_manager.cc:1398] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf: Time spent starting tablet: real 0.003s	user 0.000s	sys 0.003s
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387719 10804 catalog_manager.cc:5484] T a026f316ccae47cf8497f46a1633f13b P cdde07f1fbaf496eb82c215b5e7a0215 reported cstate change: term changed from 0 to 1, leader changed from <none> to cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131). New cstate: current_term: 1 leader_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } health_report { overall_health: HEALTHY } } }
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387851 11165 heartbeater.cc:503] Master 127.10.94.188:34325 was elected leader, sending a full tablet report...
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387871 11351 raft_consensus.cc:2781] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 1 FOLLOWER]: Leader election won for term 1
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.387974 11351 raft_consensus.cc:686] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [term 1 LEADER]: Becoming Leader. State: Replica: 686537b603c94bc3a797ce5808b5d5bf, State: Running, Role: LEADER
06:59:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.388031 11351 consensus_queue.cc:237] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } }
06:59:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.388867 10803 catalog_manager.cc:5484] T 9a6bad4b19d742f5810c6f7f0d60cdfe P 686537b603c94bc3a797ce5808b5d5bf reported cstate change: term changed from 0 to 1, leader changed from <none> to 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130). New cstate: current_term: 1 leader_uuid: "686537b603c94bc3a797ce5808b5d5bf" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } health_report { overall_health: HEALTHY } } }
06:59:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.391739 10803 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35364:
06:59:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
06:59:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:59:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "val"
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.394 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 0a0d9ac0b7ff4df4bb8bc335b40dd728
06:59:50.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.397891 10964 tablet_service.cc:1467] Processing CreateTablet for tablet 62cba0f477784f368d49193d125ab031 (DEFAULT_TABLE table=simple-test [id=0a0d9ac0b7ff4df4bb8bc335b40dd728]), partition=RANGE (key) PARTITION UNBOUNDED
06:59:50.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.398146 10964 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 62cba0f477784f368d49193d125ab031. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.400182 11356 tablet_bootstrap.cc:492] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: Bootstrap starting.
06:59:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.400640 11356 tablet_bootstrap.cc:654] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: Neither blocks nor log segments found. Creating new log.
06:59:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.400862 11356 log.cc:824] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: Log is configured to *not* fsync() on all Append() calls
06:59:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.401371 11356 tablet_bootstrap.cc:492] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: No bootstrap required, opened a new log
06:59:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.401417 11356 ts_tablet_manager.cc:1367] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: Time spent bootstrapping tablet: real 0.001s	user 0.001s	sys 0.000s
06:59:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.402782 11356 raft_consensus.cc:348] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } }
06:59:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.402880 11356 raft_consensus.cc:374] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.402896 11356 raft_consensus.cc:729] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0154f5115a5d4ca9b2e2cced3e3eb545, State: Initialized, Role: FOLLOWER
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.402958 11356 consensus_queue.cc:260] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } }
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403007 11356 raft_consensus.cc:388] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403025 11356 raft_consensus.cc:482] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403049 11356 raft_consensus.cc:3037] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Advancing to term 1
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403498 11356 raft_consensus.cc:504] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } }
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403591 11356 leader_election.cc:302] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 0154f5115a5d4ca9b2e2cced3e3eb545; no voters: 
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403699 11356 leader_election.cc:288] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 election: Requested vote from peers 
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403754 11358 raft_consensus.cc:2781] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 FOLLOWER]: Leader election won for term 1
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403852 11356 ts_tablet_manager.cc:1398] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545: Time spent starting tablet: real 0.002s	user 0.003s	sys 0.000s
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403895 11030 heartbeater.cc:503] Master 127.10.94.188:34325 was elected leader, sending a full tablet report...
06:59:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.403921 11358 raft_consensus.cc:686] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 LEADER]: Becoming Leader. State: Replica: 0154f5115a5d4ca9b2e2cced3e3eb545, State: Running, Role: LEADER
06:59:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.404000 11358 consensus_queue.cc:237] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } }
06:59:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.404752 10803 catalog_manager.cc:5484] T 62cba0f477784f368d49193d125ab031 P 0154f5115a5d4ca9b2e2cced3e3eb545 reported cstate change: term changed from 0 to 1, leader changed from <none> to 0154f5115a5d4ca9b2e2cced3e3eb545 (127.10.94.129). New cstate: current_term: 1 leader_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } health_report { overall_health: HEALTHY } } }
06:59:50.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.410439 11168 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:9a6bad4b19d742f5810c6f7f0d60cdfe)
06:59:50.411 [INFO - main] (RandomUtils.java:49) Using random seed: 1714114790411
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.412138 10803 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35364:
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "split-size-table"
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c1_i"
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c2_s"
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c3_double"
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DOUBLE
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c4_long"
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT64
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c5_bool"
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c6_short"
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT16
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c7_float"
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: FLOAT
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c8_binary"
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BINARY
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c9_unixtime_micros"
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: UNIXTIME_MICROS
06:59:50.414 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c10_byte"
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT8
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c11_decimal32"
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL32
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 9
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c12_decimal64"
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL64
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 18
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c13_decimal128"
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DECIMAL128
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       precision: 38
06:59:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c14_varchar"
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: VARCHAR
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type_attributes {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       length: 65535
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "c15_date"
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: DATE
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:59:50.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.412298 10803 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table split-size-table in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415038 11099 tablet_service.cc:1467] Processing CreateTablet for tablet 30532dbf58484952b9456afd5e75113f (DEFAULT_TABLE table=split-size-table [id=8dac1bb306d340ce8d8b0079e1725d62]), partition=RANGE (key) PARTITION UNBOUNDED
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415179 11099 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30532dbf58484952b9456afd5e75113f. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415153 11234 tablet_service.cc:1467] Processing CreateTablet for tablet 30532dbf58484952b9456afd5e75113f (DEFAULT_TABLE table=split-size-table [id=8dac1bb306d340ce8d8b0079e1725d62]), partition=RANGE (key) PARTITION UNBOUNDED
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415267 11234 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30532dbf58484952b9456afd5e75113f. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415691 10964 tablet_service.cc:1467] Processing CreateTablet for tablet 30532dbf58484952b9456afd5e75113f (DEFAULT_TABLE table=split-size-table [id=8dac1bb306d340ce8d8b0079e1725d62]), partition=RANGE (key) PARTITION UNBOUNDED
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.415791 10964 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 30532dbf58484952b9456afd5e75113f. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.416579 11342 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215: Bootstrap starting.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.416662 11343 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf: Bootstrap starting.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.416994 11342 tablet_bootstrap.cc:654] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215: Neither blocks nor log segments found. Creating new log.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417025 11356 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545: Bootstrap starting.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417062 11343 tablet_bootstrap.cc:654] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf: Neither blocks nor log segments found. Creating new log.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417456 11356 tablet_bootstrap.cc:654] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545: Neither blocks nor log segments found. Creating new log.
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417675 11342 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215: No bootstrap required, opened a new log
06:59:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417735 11342 ts_tablet_manager.cc:1367] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215: Time spent bootstrapping tablet: real 0.001s	user 0.001s	sys 0.000s
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417898 11342 raft_consensus.cc:348] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417954 11342 raft_consensus.cc:374] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.417976 11342 raft_consensus.cc:729] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdde07f1fbaf496eb82c215b5e7a0215, State: Initialized, Role: FOLLOWER
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418011 11343 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf: No bootstrap required, opened a new log
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418017 11342 consensus_queue.cc:260] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418057 11343 ts_tablet_manager.cc:1367] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf: Time spent bootstrapping tablet: real 0.001s	user 0.000s	sys 0.001s
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418113 11356 tablet_bootstrap.cc:492] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545: No bootstrap required, opened a new log
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418125 11342 ts_tablet_manager.cc:1398] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215: Time spent starting tablet: real 0.000s	user 0.001s	sys 0.000s
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418154 11356 ts_tablet_manager.cc:1367] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545: Time spent bootstrapping tablet: real 0.001s	user 0.001s	sys 0.000s
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418221 11343 raft_consensus.cc:348] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418277 11343 raft_consensus.cc:374] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418296 11343 raft_consensus.cc:729] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 686537b603c94bc3a797ce5808b5d5bf, State: Initialized, Role: FOLLOWER
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418274 11356 raft_consensus.cc:348] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418392 11356 raft_consensus.cc:374] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418359 11343 consensus_queue.cc:260] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418416 11356 raft_consensus.cc:729] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0154f5115a5d4ca9b2e2cced3e3eb545, State: Initialized, Role: FOLLOWER
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418462 11343 ts_tablet_manager.cc:1398] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf: Time spent starting tablet: real 0.000s	user 0.000s	sys 0.001s
06:59:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.418475 11356 consensus_queue.cc:260] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.419171 11356 ts_tablet_manager.cc:1398] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545: Time spent starting tablet: real 0.001s	user 0.001s	sys 0.000s
06:59:50.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.527725 11303 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:30532dbf58484952b9456afd5e75113f)
06:59:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.537482 11033 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:30532dbf58484952b9456afd5e75113f)
06:59:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.614229 11358 raft_consensus.cc:482] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.614344 11358 raft_consensus.cc:504] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.614706 11358 leader_election.cc:288] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471), cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695)
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618342 11254 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30532dbf58484952b9456afd5e75113f" candidate_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" is_pre_election: true
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618333 11119 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30532dbf58484952b9456afd5e75113f" candidate_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "686537b603c94bc3a797ce5808b5d5bf" is_pre_election: true
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618485 11119 raft_consensus.cc:2454] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0154f5115a5d4ca9b2e2cced3e3eb545 in term 0.
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618485 11254 raft_consensus.cc:2454] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0154f5115a5d4ca9b2e2cced3e3eb545 in term 0.
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618710 10918 leader_election.cc:302] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0154f5115a5d4ca9b2e2cced3e3eb545, cdde07f1fbaf496eb82c215b5e7a0215; no voters: 
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618834 11358 raft_consensus.cc:2781] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618885 11358 raft_consensus.cc:482] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:59:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.618906 11358 raft_consensus.cc:3037] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 0 FOLLOWER]: Advancing to term 1
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619470 11358 raft_consensus.cc:504] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619596 11358 leader_election.cc:288] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 election: Requested vote from peers 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471), cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695)
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619731 11119 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30532dbf58484952b9456afd5e75113f" candidate_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "686537b603c94bc3a797ce5808b5d5bf"
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619731 11254 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "30532dbf58484952b9456afd5e75113f" candidate_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cdde07f1fbaf496eb82c215b5e7a0215"
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619804 11119 raft_consensus.cc:3037] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 0 FOLLOWER]: Advancing to term 1
06:59:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.619820 11254 raft_consensus.cc:3037] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 0 FOLLOWER]: Advancing to term 1
06:59:50.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.620668 11254 raft_consensus.cc:2454] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0154f5115a5d4ca9b2e2cced3e3eb545 in term 1.
06:59:50.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.620828 10918 leader_election.cc:302] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0154f5115a5d4ca9b2e2cced3e3eb545, cdde07f1fbaf496eb82c215b5e7a0215; no voters: 
06:59:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.620914 11358 raft_consensus.cc:2781] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 FOLLOWER]: Leader election won for term 1
06:59:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.620956 11119 raft_consensus.cc:2454] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0154f5115a5d4ca9b2e2cced3e3eb545 in term 1.
06:59:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.620971 11358 raft_consensus.cc:686] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [term 1 LEADER]: Becoming Leader. State: Replica: 0154f5115a5d4ca9b2e2cced3e3eb545, State: Running, Role: LEADER
06:59:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.621023 11358 consensus_queue.cc:237] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } }
06:59:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.621562 10803 catalog_manager.cc:5484] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 reported cstate change: term changed from 0 to 1, leader changed from <none> to 0154f5115a5d4ca9b2e2cced3e3eb545 (127.10.94.129). New cstate: current_term: 1 leader_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0154f5115a5d4ca9b2e2cced3e3eb545" member_type: VOTER last_known_addr { host: "127.10.94.129" port: 36247 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 } health_report { overall_health: UNKNOWN } } }
06:59:50.637 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
06:59:50.637 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
06:59:50.637 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=1), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [1ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=null) after lookup
06:59:50.638 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
06:59:50.639 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] Successfully connected to peer
06:59:50.648 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:50.649 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc6c3ff56, L:/127.0.0.1:51122 - R:/127.10.94.129:36247] using TOKEN/null
06:59:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.653965 11254 raft_consensus.cc:1261] T 30532dbf58484952b9456afd5e75113f P cdde07f1fbaf496eb82c215b5e7a0215 [term 1 FOLLOWER]: Refusing update from remote peer 0154f5115a5d4ca9b2e2cced3e3eb545: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.654078 11119 raft_consensus.cc:1261] T 30532dbf58484952b9456afd5e75113f P 686537b603c94bc3a797ce5808b5d5bf [term 1 FOLLOWER]: Refusing update from remote peer 0154f5115a5d4ca9b2e2cced3e3eb545: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.654429 11358 consensus_queue.cc:1035] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [LEADER]: Connected to new peer: Peer: permanent_uuid: "686537b603c94bc3a797ce5808b5d5bf" member_type: VOTER last_known_addr { host: "127.10.94.130" port: 33471 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.654608 11360 consensus_queue.cc:1035] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cdde07f1fbaf496eb82c215b5e7a0215" member_type: VOTER last_known_addr { host: "127.10.94.131" port: 46695 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:50.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.658372 11354 mvcc.cc:205] Tried to move back new op lower bound from 7021014182515924992 to 7021014182384209920. Current Snapshot: MvccSnapshot[applied={T|T < 7021014182515924992}]
06:59:50.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.658680 11362 mvcc.cc:205] Tried to move back new op lower bound from 7021014182515924992 to 7021014182384209920. Current Snapshot: MvccSnapshot[applied={T|T < 7021014182515924992 or (T in {7021014182515924992})}]
06:59:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.678021 10874 catalog_manager.cc:1259] Loaded cluster ID: bf8fc0df8d204989893c99206ed3b73e
06:59:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.678095 10874 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: loading cluster ID for follower catalog manager: success
06:59:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.679354 10874 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: acquiring CA information for follower catalog manager: success
06:59:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.679718 10874 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7602761493674efcb4c167e3abd06416: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.689368 10888 catalog_manager.cc:1259] Loaded cluster ID: bf8fc0df8d204989893c99206ed3b73e
06:59:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.689420 10888 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: loading cluster ID for follower catalog manager: success
06:59:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.690274 10888 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: acquiring CA information for follower catalog manager: success
06:59:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.690608 10888 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c18b20b95cad4b4a8ede1f4f887d0f72: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.778177 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.000000
06:59:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.787952 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.000000
06:59:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.811791 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.033s	user 0.008s	sys 0.000s Metrics: {"bytes_written":39749,"cfile_init":1,"compiler_manager_pool.queue_time_us":126,"drs_written":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4,"lbm_write_time_us":516,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":44,"thread_start_us":152,"threads_started":2}
06:59:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.812453 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 613 bytes on disk
06:59:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.812760 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:59:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.815783 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.028s	user 0.000s	sys 0.007s Metrics: {"bytes_written":39764,"cfile_init":1,"compiler_manager_pool.queue_time_us":2078,"drs_written":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4,"lbm_write_time_us":528,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":47,"thread_start_us":157,"threads_started":2}
06:59:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.815985 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 646 bytes on disk
06:59:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.816148 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":20,"lbm_reads_lt_1ms":4}
06:59:50.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.911052 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953437
06:59:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.948432 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.037s	user 0.001s	sys 0.007s Metrics: {"bytes_written":40195,"cfile_init":1,"compiler_manager_pool.queue_time_us":772,"drs_written":1,"lbm_read_time_us":26,"lbm_reads_lt_1ms":4,"lbm_write_time_us":504,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":74,"thread_start_us":156,"threads_started":2}
06:59:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.948894 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 947 bytes on disk
06:59:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.950317 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":26,"lbm_reads_lt_1ms":4}
06:59:51.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.063166 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.000000
06:59:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.066495 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.000000
06:59:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.067404 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.004s	sys 0.000s Metrics: {"bytes_written":40035,"cfile_init":1,"dirs.queue_time_us":614,"drs_written":1,"lbm_read_time_us":18,"lbm_reads_lt_1ms":4,"lbm_write_time_us":599,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":57}
06:59:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.067539 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 770 bytes on disk
06:59:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.067641 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4}
06:59:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.070468 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.000s	sys 0.003s Metrics: {"bytes_written":39941,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":131,"dirs.run_wall_time_us":1016,"drs_written":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4,"lbm_write_time_us":573,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":54}
06:59:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.070770 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 736 bytes on disk
06:59:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.070967 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.200747 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953496
06:59:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.204464 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40234,"cfile_init":1,"dirs.queue_time_us":48,"drs_written":1,"lbm_read_time_us":9,"lbm_reads_lt_1ms":4,"lbm_write_time_us":448,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":65}
06:59:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.204597 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 858 bytes on disk
06:59:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.205929 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:59:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.317860 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.321283 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.322744 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.005s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40310,"cfile_init":1,"dirs.queue_time_us":33,"drs_written":1,"lbm_read_time_us":25,"lbm_reads_lt_1ms":4,"lbm_write_time_us":576,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":75}
06:59:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.322878 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 959 bytes on disk
06:59:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.322973 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.325035 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40351,"cfile_init":1,"dirs.queue_time_us":47,"dirs.run_cpu_time_us":106,"dirs.run_wall_time_us":1046,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":601,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":77}
06:59:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.325163 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 983 bytes on disk
06:59:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.325275 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.456445 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.460063 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40398,"cfile_init":1,"dirs.queue_time_us":50,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":419,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":84}
06:59:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.460247 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1050 bytes on disk
06:59:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.460480 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":24,"lbm_reads_lt_1ms":4}
06:59:51.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.573208 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953555
06:59:51.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.575510 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953555
06:59:51.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.577128 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.001s	sys 0.003s Metrics: {"bytes_written":40478,"cfile_init":1,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":631,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":84}
06:59:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.579324 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1050 bytes on disk
06:59:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.579752 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:59:51.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.580106 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.000s	sys 0.003s Metrics: {"bytes_written":40531,"cfile_init":1,"dirs.queue_time_us":40,"dirs.run_cpu_time_us":142,"dirs.run_wall_time_us":1198,"drs_written":1,"lbm_read_time_us":15,"lbm_reads_lt_1ms":4,"lbm_write_time_us":552,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":82}
06:59:51.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.580248 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1031 bytes on disk
06:59:51.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.580374 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:59:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.710769 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.714732 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.001s	sys 0.002s Metrics: {"bytes_written":40564,"cfile_init":1,"dirs.queue_time_us":53,"dirs.run_cpu_time_us":148,"dirs.run_wall_time_us":1494,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":425,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":86}
06:59:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.714946 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1071 bytes on disk
06:59:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.715068 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:51.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.830216 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953555
06:59:51.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.830623 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953555
06:59:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.833730 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.000s	sys 0.002s Metrics: {"bytes_written":40519,"cfile_init":1,"dirs.queue_time_us":45,"drs_written":1,"lbm_read_time_us":11,"lbm_reads_lt_1ms":4,"lbm_write_time_us":422,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":89}
06:59:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.833868 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1105 bytes on disk
06:59:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.833983 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.834000 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.000s	sys 0.003s Metrics: {"bytes_written":40519,"cfile_init":1,"dirs.queue_time_us":591,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":565,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":89}
06:59:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.834141 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1105 bytes on disk
06:59:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.834270 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.965286 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.969257 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.002s	sys 0.001s Metrics: {"bytes_written":40521,"cfile_init":1,"dirs.queue_time_us":585,"drs_written":1,"lbm_read_time_us":15,"lbm_reads_lt_1ms":4,"lbm_write_time_us":514,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":84}
06:59:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.969406 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1048 bytes on disk
06:59:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:51.969635 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.084194 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.086787 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.087751 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.003s	sys 0.001s Metrics: {"bytes_written":40509,"cfile_init":1,"dirs.queue_time_us":68,"dirs.run_cpu_time_us":137,"dirs.run_wall_time_us":1413,"drs_written":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4,"lbm_write_time_us":458,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":87}
06:59:52.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.089236 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1083 bytes on disk
06:59:52.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.089491 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4}
06:59:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.090034 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.002s	sys 0.001s Metrics: {"bytes_written":40509,"cfile_init":1,"dirs.queue_time_us":37,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":435,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":87}
06:59:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.090186 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1083 bytes on disk
06:59:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.090893 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:52.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.219933 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.223765 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.002s	sys 0.001s Metrics: {"bytes_written":40493,"cfile_init":1,"dirs.queue_time_us":35,"dirs.run_cpu_time_us":141,"dirs.run_wall_time_us":1451,"drs_written":1,"lbm_read_time_us":11,"lbm_reads_lt_1ms":4,"lbm_write_time_us":465,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":91}
06:59:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.224085 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1124 bytes on disk
06:59:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.224336 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":18,"lbm_reads_lt_1ms":4}
06:59:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.339839 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953555
06:59:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.341221 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.343673 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40610,"cfile_init":1,"dirs.queue_time_us":21,"dirs.run_cpu_time_us":149,"dirs.run_wall_time_us":1266,"drs_written":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4,"lbm_write_time_us":536,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":90}
06:59:52.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.343828 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1114 bytes on disk
06:59:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.343959 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4}
06:59:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.344632 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40610,"cfile_init":1,"dirs.queue_time_us":37,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":487,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":90}
06:59:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.344763 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1114 bytes on disk
06:59:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.344844 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":13,"lbm_reads_lt_1ms":4}
06:59:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.474627 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.478974 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.002s	sys 0.001s Metrics: {"bytes_written":40578,"cfile_init":1,"dirs.queue_time_us":49,"dirs.run_cpu_time_us":161,"dirs.run_wall_time_us":1379,"drs_written":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4,"lbm_write_time_us":579,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":93}
06:59:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.479370 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1146 bytes on disk
06:59:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.479604 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":25,"lbm_reads_lt_1ms":4}
06:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.594188 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.595088 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.598920 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.001s Metrics: {"bytes_written":40543,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":114,"dirs.run_wall_time_us":1156,"drs_written":1,"lbm_read_time_us":11,"lbm_reads_lt_1ms":4,"lbm_write_time_us":535,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":96}
06:59:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.599254 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1173 bytes on disk
06:59:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.598954 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.005s	user 0.004s	sys 0.000s Metrics: {"bytes_written":40543,"cfile_init":1,"dirs.queue_time_us":45,"dirs.run_cpu_time_us":164,"dirs.run_wall_time_us":1294,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":613,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":96}
06:59:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.599457 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1173 bytes on disk
06:59:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.599586 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":18,"lbm_reads_lt_1ms":4}
06:59:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.600368 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4}
06:59:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.729935 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:52.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.733587 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40696,"cfile_init":1,"dirs.queue_time_us":640,"drs_written":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4,"lbm_write_time_us":566,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":94}
06:59:52.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.733757 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1156 bytes on disk
06:59:52.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.733870 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":15,"lbm_reads_lt_1ms":4}
06:59:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.850150 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.850661 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854110 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.000s	sys 0.003s Metrics: {"bytes_written":40674,"cfile_init":1,"dirs.queue_time_us":322,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":538,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":95}
06:59:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854413 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1166 bytes on disk
06:59:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854667 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.004s	sys 0.000s Metrics: {"bytes_written":40674,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":132,"dirs.run_wall_time_us":1320,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":542,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":95}
06:59:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854831 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1166 bytes on disk
06:59:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854933 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.854972 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:52.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.984141 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.987666 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40769,"cfile_init":1,"dirs.queue_time_us":47,"drs_written":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4,"lbm_write_time_us":435,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":101}
06:59:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.987862 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1229 bytes on disk
06:59:52.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:52.987970 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4}
06:59:53.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.105278 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:53.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.105331 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.108952 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40776,"cfile_init":1,"dirs.queue_time_us":52,"drs_written":1,"lbm_read_time_us":10,"lbm_reads_lt_1ms":4,"lbm_write_time_us":455,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":103}
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.109112 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1249 bytes on disk
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.109136 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.003s	sys 0.000s Metrics: {"bytes_written":40776,"cfile_init":1,"dirs.queue_time_us":32,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":455,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":103}
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.109278 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1249 bytes on disk
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.109270 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:59:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.109539 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:59:53.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.238268 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.241614 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.002s	sys 0.000s Metrics: {"bytes_written":40652,"cfile_init":1,"dirs.queue_time_us":43,"drs_written":1,"lbm_read_time_us":11,"lbm_reads_lt_1ms":4,"lbm_write_time_us":393,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":97}
06:59:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.241782 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1190 bytes on disk
06:59:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.241945 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4}
06:59:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.359508 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.359926 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945743
06:59:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.363500 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.000s	sys 0.003s Metrics: {"bytes_written":40621,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":126,"dirs.run_wall_time_us":1221,"drs_written":1,"lbm_read_time_us":12,"lbm_reads_lt_1ms":4,"lbm_write_time_us":481,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":93}
06:59:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.363754 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1145 bytes on disk
06:59:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.363866 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.364459 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.005s	user 0.002s	sys 0.002s Metrics: {"bytes_written":40621,"cfile_init":1,"dirs.queue_time_us":26,"dirs.run_cpu_time_us":166,"dirs.run_wall_time_us":1304,"drs_written":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4,"lbm_write_time_us":517,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":93}
06:59:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.364758 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1145 bytes on disk
06:59:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.364900 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":20,"lbm_reads_lt_1ms":4}
06:59:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.492223 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=2.945654
06:59:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.496704 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.004s	user 0.004s	sys 0.000s Metrics: {"bytes_written":40631,"cfile_init":1,"dirs.queue_time_us":17,"dirs.run_cpu_time_us":179,"dirs.run_wall_time_us":1431,"drs_written":1,"lbm_read_time_us":12,"lbm_reads_lt_1ms":4,"lbm_write_time_us":599,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":93}
06:59:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.496871 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1145 bytes on disk
06:59:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.497295 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:59:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.614146 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.615234 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.953466
06:59:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.617362 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.000s	sys 0.002s Metrics: {"bytes_written":40461,"cfile_init":1,"dirs.queue_time_us":54,"drs_written":1,"lbm_read_time_us":13,"lbm_reads_lt_1ms":4,"lbm_write_time_us":490,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":87}
06:59:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.617622 11303 maintenance_manager.cc:419] P cdde07f1fbaf496eb82c215b5e7a0215: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1078 bytes on disk
06:59:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.617822 11191 maintenance_manager.cc:643] P cdde07f1fbaf496eb82c215b5e7a0215: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:59:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.618602 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.002s	sys 0.001s Metrics: {"bytes_written":40461,"cfile_init":1,"dirs.queue_time_us":40,"dirs.run_cpu_time_us":134,"dirs.run_wall_time_us":1249,"drs_written":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4,"lbm_write_time_us":490,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":87}
06:59:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.618748 11033 maintenance_manager.cc:419] P 0154f5115a5d4ca9b2e2cced3e3eb545: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 1078 bytes on disk
06:59:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.618923 10921 maintenance_manager.cc:643] P 0154f5115a5d4ca9b2e2cced3e3eb545: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":17,"lbm_reads_lt_1ms":4}
06:59:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.747639 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling FlushMRSOp(30532dbf58484952b9456afd5e75113f): perf score=1.000000
06:59:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.750450 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: FlushMRSOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.003s	user 0.002s	sys 0.000s Metrics: {"bytes_written":39798,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":161,"dirs.run_wall_time_us":1265,"drs_written":1,"lbm_read_time_us":12,"lbm_reads_lt_1ms":4,"lbm_write_time_us":427,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
06:59:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.750617 11168 maintenance_manager.cc:419] P 686537b603c94bc3a797ce5808b5d5bf: Scheduling UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f): 547 bytes on disk
06:59:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.750749 11056 maintenance_manager.cc:643] P 686537b603c94bc3a797ce5808b5d5bf: UndoDeltaBlockGCOp(30532dbf58484952b9456afd5e75113f) complete. Timing: real 0.000s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:59:58.594 [INFO - main] (TestKuduBackup.scala:963) nowMs: 1714114798594
06:59:58.594 [INFO - main] (TestKuduBackup.scala:965) propagated physicalMicros: 1714114793591891
06:59:58.594 [INFO - main] (TestKuduBackup.scala:966) propagated logical: 0
06:59:58.594 [INFO - main] (KuduBackup.scala:111) Backing up to root path: file:///tmp/backup2979075306664062291/
06:59:58.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
06:59:58.597 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:55) Backing up table split-size-table to path: file:/tmp/backup2979075306664062291/8dac1bb306d340ce8d8b0079e1725d62-split-size-table/1714114798594
06:59:58.597 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:66) Looking for a previous backup: forceFull and fromMs options are not set.
06:59:58.598 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:73) No previous backup was found. Starting a full backup.
06:59:58.598 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-tasklQverR/java/build/dist-test/spark-warehouse').
06:59:58.598 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-tasklQverR/java/build/dist-test/spark-warehouse'.
06:59:58.615 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:58.616 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:58.616 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:58.616 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:58.616 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:58.616 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:58.617 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for split-size-table
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, 0x8000002E), 35584, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000002E, 0x80000064), 35761, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000064, 0x800000B1), 36171, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000B1, 0x80000103), 36351, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.642 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000103, 0x8000015C), 36339, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000015C, 0x800001B3), 36329, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001B3, 0x8000020D), 36430, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000020D, 0x8000026D), 36363, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000026D, 0x800002CC), 36494, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002CC, 0x80000333), 36596, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000333, 0x80000390), 36441, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.643 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000390, <end>), 36281, "30532dbf58484952b9456afd5e75113f" [<start>, <end>)
06:59:58.657 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Starting job: save at KuduBackup.scala:97
06:59:58.657 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (save at KuduBackup.scala:97) with 12 output partitions
06:59:58.657 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (save at KuduBackup.scala:97)
06:59:58.657 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
06:59:58.658 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
06:59:58.658 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97), which has no missing parents
06:59:58.673 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 189.4 KiB, free 1993.7 MiB)
06:59:58.675 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 66.7 KiB, free 1993.6 MiB)
06:59:58.675 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-str9.c.gcp-upstream.internal:45795 (size: 66.7 KiB, free: 1993.8 MiB)
06:59:58.676 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
06:59:58.676 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 12 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11))
06:59:58.676 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 12 tasks resource profile 0
06:59:58.677 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-str9.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5291 bytes) taskResourceAssignments Map()
06:59:58.678 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
06:59:58.687 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:58.688 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:58.689 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:376) Parquet block size to 134217728
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:377) Parquet page size to 1048576
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:378) Parquet dictionary page size to 1048576
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:379) Dictionary is on
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:380) Validation is off
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:381) Writer version is: PARQUET_1_0
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:382) Maximum row group padding size is 8388608 bytes
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:383) Page size checking is: estimated
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:384) Min row count for page size check is: 100
06:59:58.690 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:385) Max row count for page size check is: 10000
06:59:58.691 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Initialized Parquet WriteSupport with Catalyst schema:
{
  "type" : "struct",
  "fields" : [ {
    "name" : "key",
    "type" : "integer",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c1_i",
    "type" : "integer",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c2_s",
    "type" : "string",
    "nullable" : true,
    "metadata" : { }
  }, {
    "name" : "c3_double",
    "type" : "double",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c4_long",
    "type" : "long",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c5_bool",
    "type" : "boolean",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c6_short",
    "type" : "short",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c7_float",
    "type" : "float",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c8_binary",
    "type" : "binary",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c9_unixtime_micros",
    "type" : "timestamp",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c10_byte",
    "type" : "byte",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c11_decimal32",
    "type" : "decimal(9,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c12_decimal64",
    "type" : "decimal(18,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c13_decimal128",
    "type" : "decimal(38,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c14_varchar",
    "type" : "string",
    "nullable" : true,
    "metadata" : { }
  }, {
    "name" : "c15_date",
    "type" : "date",
    "nullable" : false,
    "metadata" : { }
  } ]
}
and corresponding Parquet message type:
message spark_schema {
  required int32 key;
  required int32 c1_i;
  optional binary c2_s (UTF8);
  required double c3_double;
  required int64 c4_long;
  required boolean c5_bool;
  required int32 c6_short (INT_16);
  required float c7_float;
  required binary c8_binary;
  required int96 c9_unixtime_micros;
  required int32 c10_byte (INT_8);
  required int32 c11_decimal32 (DECIMAL(9,0));
  required int64 c12_decimal64 (DECIMAL(18,0));
  required fixed_len_byte_array(16) c13_decimal128 (DECIMAL(38,0));
  optional binary c14_varchar (UTF8);
  required int32 c15_date (DATE);
}

       
06:59:58.700 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
06:59:58.701 [DEBUG - kudu-nio-0] (Connection.java:757) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] Successfully connected to peer
06:59:58.708 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:58.709 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9b76d73d, L:/127.0.0.1:35504 - R:/127.10.94.131:46695] using TOKEN/null
06:59:58.965 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 46 rows, more = false, responseScanTimestamp = 7021014215041024000) for scanner KuduScanner(table=split-size-table, tablet=30532dbf58484952b9456afd5e75113f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x8000002E, lastPrimaryKey=<last>)
06:59:59.008 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (InternalParquetRecordWriter.java:165) Flushing mem columnStore to file. allocated memory: 40743
06:59:59.011 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:593) Saved output of task 'attempt_20240426065958121977045308021280_0000_m_000000_0' to file:/tmp/backup2979075306664062291/8dac1bb306d340ce8d8b0079e1725d62-split-size-table/1714114798594/_temporary/0/task_20240426065958121977045308021280_0000_m_000000
06:59:59.011 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) attempt_20240426065958121977045308021280_0000_m_000000_0: Committed
06:59:59.012 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 6343 bytes result sent to driver
06:59:59.013 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-str9.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5297 bytes) taskResourceAssignments Map()
06:59:59.013 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
06:59:59.013 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 336 ms on dist-test-slave-str9.c.gcp-upstream.internal (executor driver) (1/12)
06:59:59.021 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:59.022 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:59.023 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:59.040 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 54 rows, more = false, responseScanTimestamp = 7021014215041024000) for scanner KuduScanner(table=split-size-table, tablet=30532dbf58484952b9456afd5e75113f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=0x8000002E, endPrimaryKey=0x80000064, lastPrimaryKey=<last>)
06:59:59.047 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (CodecConfig.java:95) Compression: SNAPPY
06:59:59.047 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (CodecConfig.java:95) Compression: SNAPPY
06:59:59.047 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:376) Parquet block size to 134217728
06:59:59.047 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:377) Parquet page size to 1048576
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:378) Parquet dictionary page size to 1048576
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:379) Dictionary is on
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:380) Validation is off
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:381) Writer version is: PARQUET_1_0
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:382) Maximum row group padding size is 8388608 bytes
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:383) Page size checking is: estimated
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:384) Min row count for page size check is: 100
06:59:59.048 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (ParquetOutputFormat.java:385) Max row count for page size check is: 10000
06:59:59.049 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Initialized Parquet WriteSupport with Catalyst schema:
{
  "type" : "struct",
  "fields" : [ {
    "name" : "key",
    "type" : "integer",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c1_i",
    "type" : "integer",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c2_s",
    "type" : "string",
    "nullable" : true,
    "metadata" : { }
  }, {
    "name" : "c3_double",
    "type" : "double",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c4_long",
    "type" : "long",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c5_bool",
    "type" : "boolean",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c6_short",
    "type" : "short",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c7_float",
    "type" : "float",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c8_binary",
    "type" : "binary",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c9_unixtime_micros",
    "type" : "timestamp",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c10_byte",
    "type" : "byte",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c11_decimal32",
    "type" : "decimal(9,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c12_decimal64",
    "type" : "decimal(18,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c13_decimal128",
    "type" : "decimal(38,0)",
    "nullable" : false,
    "metadata" : { }
  }, {
    "name" : "c14_varchar",
    "type" : "string",
    "nullable" : true,
    "metadata" : { }
  }, {
    "name" : "c15_date",
    "type" : "date",
    "nullable" : false,
    "metadata" : { }
  } ]
}
and corresponding Parquet message type:
message spark_schema {
  required int32 key;
  required int32 c1_i;
  optional binary c2_s (UTF8);
  required double c3_double;
  required int64 c4_long;
  required boolean c5_bool;
  required int32 c6_short (INT_16);
  required float c7_float;
  required binary c8_binary;
  required int96 c9_unixtime_micros;
  required int32 c10_byte (INT_8);
  required int32 c11_decimal32 (DECIMAL(9,0));
  required int64 c12_decimal64 (DECIMAL(18,0));
  required fixed_len_byte_array(16) c13_decimal128 (DECIMAL(38,0));
  optional binary c14_varchar (UTF8);
  required int32 c15_date (DATE);
}

       
06:59:59.081 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (InternalParquetRecordWriter.java:165) Flushing mem columnStore to file. allocated memory: 41774
06:59:59.083 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:593) Saved output of task 'attempt_202404260659583218930017726809948_0000_m_000001_1' to file:/tmp/backup2979075306664062291/8dac1bb306d340ce8d8b0079e1725d62-split-size-table/1714114798594/_temporary/0/task_202404260659583218930017726809948_0000_m_000001
06:59:59.083 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) attempt_202404260659583218930017726809948_0000_m_000001_1: Committed
06:59:59.083 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1). 6300 bytes result sent to driver
06:59:59.084 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5297 bytes) taskResourceAssignments Map()
06:59:59.084 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
06:59:59.085 [INFO - task-result-getter-1] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1) in 73 ms on dist-test-slave-str9.c.gcp-upstream.internal (executor driver) (2/12)
06:59:59.092 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:59.093 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
06:59:59.094 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
06:59:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714114799 (unix time) try "date -d @1714114799" if you are using GNU date ***
06:59:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @                0x0 (unknown)
06:59:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 11172 (TID 0x7f6f73281700) from PID 8; stack trace: ***
06:59:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f90a07980 (unknown)
06:59:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f908a368e kudu::ScanSpec::SetLowerBoundKey()
06:59:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f9133c2c3 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
06:59:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f91343614 kudu::tablet::CFileSet::Iterator::Init()
06:59:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f913d2a9b kudu::tablet::DeltaApplier::Init()
06:59:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f90858ef2 kudu::MaterializingIterator::Init()
06:59:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f9084fc80 kudu::InitAndMaybeWrap()
06:59:59.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f90851995 kudu::UnionIterator::InitSubIterators()
06:59:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f908543fa kudu::UnionIterator::Init()
06:59:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f912de5c7 kudu::tablet::Tablet::Iterator::Init()
06:59:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f916bab0d kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
06:59:59.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f916bcb0f kudu::tserver::TabletServiceImpl::Scan()
06:59:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f90ca46f8 kudu::rpc::GeneratedServiceIf::Handle()
06:59:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f90ca56cc kudu::rpc::ServicePool::RunThread()
06:59:59.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f906da394 kudu::Thread::SuperviseThread()
06:59:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f909fc6db start_thread
06:59:59.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f6f8eb4071f clone
06:59:59.266 [DEBUG - kudu-nio-0] (Connection.java:239) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] handling channelInactive
06:59:59.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state READY due to: connection closed
06:59:59.266 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: connection closed
06:59:59.271 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
06:59:59.271 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] Successfully connected to peer
06:59:59.279 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:59:59.280 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x31937954, L:/127.0.0.1:51694 - R:/127.10.94.130:33471] using TOKEN/null
06:59:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714114799 (unix time) try "date -d @1714114799" if you are using GNU date ***
06:59:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @                0x0 (unknown)
06:59:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 11037 (TID 0x7f5fd896a700) from PID 8; stack trace: ***
06:59:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff60f0980 (unknown)
06:59:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5f8c68e kudu::ScanSpec::SetLowerBoundKey()
06:59:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff6a252c3 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
06:59:59.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff6a2c614 kudu::tablet::CFileSet::Iterator::Init()
06:59:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff6abba9b kudu::tablet::DeltaApplier::Init()
06:59:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5f41ef2 kudu::MaterializingIterator::Init()
06:59:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5f38c80 kudu::InitAndMaybeWrap()
06:59:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5f3a995 kudu::UnionIterator::InitSubIterators()
06:59:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5f3d3fa kudu::UnionIterator::Init()
06:59:59.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff69c75c7 kudu::tablet::Tablet::Iterator::Init()
06:59:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff6da3b0d kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
06:59:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff6da5b0f kudu::tserver::TabletServiceImpl::Scan()
06:59:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff638d6f8 kudu::rpc::GeneratedServiceIf::Handle()
06:59:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff638e6cc kudu::rpc::ServicePool::RunThread()
06:59:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff5dc3394 kudu::Thread::SuperviseThread()
06:59:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff60e56db start_thread
06:59:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f5ff422971f clone
06:59:59.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:59.353696 10918 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.94.131:46695: connect: Connection refused (error 111)
06:59:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:59.354151 10918 consensus_peers.cc:485] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 -> Peer cdde07f1fbaf496eb82c215b5e7a0215 (127.10.94.131:46695): Couldn't send request to peer cdde07f1fbaf496eb82c215b5e7a0215. Status: Network error: Client connection negotiation failed: client connection to 127.10.94.131:46695: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:59:59.449 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] handling channelInactive
06:59:59.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state READY due to: connection closed
06:59:59.449 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: connection closed
06:59:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714114799 (unix time) try "date -d @1714114799" if you are using GNU date ***
06:59:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @                0x0 (unknown)
06:59:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 10902 (TID 0x7f9a47c65700) from PID 8; stack trace: ***
06:59:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a653eb980 (unknown)
06:59:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a6528768e kudu::ScanSpec::SetLowerBoundKey()
06:59:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65d202c3 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
06:59:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65d27614 kudu::tablet::CFileSet::Iterator::Init()
06:59:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65db6a9b kudu::tablet::DeltaApplier::Init()
06:59:59.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a6523cef2 kudu::MaterializingIterator::Init()
06:59:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65233c80 kudu::InitAndMaybeWrap()
06:59:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65235995 kudu::UnionIterator::InitSubIterators()
06:59:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a652383fa kudu::UnionIterator::Init()
06:59:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a65cc25c7 kudu::tablet::Tablet::Iterator::Init()
06:59:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a6609eb0d kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
06:59:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a660a0b0f kudu::tserver::TabletServiceImpl::Scan()
06:59:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a656886f8 kudu::rpc::GeneratedServiceIf::Handle()
06:59:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a656896cc kudu::rpc::ServicePool::RunThread()
06:59:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a650be394 kudu::Thread::SuperviseThread()
06:59:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:59.504010 10917 consensus_peers.cc:485] T 30532dbf58484952b9456afd5e75113f P 0154f5115a5d4ca9b2e2cced3e3eb545 -> Peer 686537b603c94bc3a797ce5808b5d5bf (127.10.94.130:33471): Couldn't send request to peer 686537b603c94bc3a797ce5808b5d5bf. Status: Network error: Client connection negotiation failed: client connection to 127.10.94.130:33471: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:59:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a653e06db start_thread
06:59:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     @     0x7f9a6352471f clone
06:59:59.634 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] handling channelInactive
06:59:59.635 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state READY due to: connection closed
06:59:59.635 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: connection closed
06:59:59.651 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
06:59:59.651 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
06:59:59.652 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=4, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=4, TimeoutTracker(timeout=30000, elapsed=557), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2)))) after lookup
06:59:59.652 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
06:59:59.653 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.654 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.654 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.670 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
06:59:59.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
06:59:59.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
06:59:59.671 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
06:59:59.730 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
06:59:59.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
06:59:59.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
06:59:59.731 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
06:59:59.771 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
06:59:59.771 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
06:59:59.772 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=8, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=8, TimeoutTracker(timeout=30000, elapsed=677), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559)))) after lookup
06:59:59.772 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
06:59:59.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.773 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
06:59:59.850 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
06:59:59.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
06:59:59.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
06:59:59.852 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:00.691 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:00.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:00.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:00.692 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:01.991 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:01.991 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:01.991 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=12, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=12, TimeoutTracker(timeout=30000, elapsed=2897), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207)))) after lookup
07:00:01.993 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:01.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:01.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:01.993 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:04.370 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:04.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:04.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:04.372 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:06.730 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:06.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:06.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:06.731 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:07.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:07.051 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:07.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=16, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=16, TimeoutTracker(timeout=30000, elapsed=7957), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744)))) after lookup
07:00:07.053 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:07.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:07.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:07.054 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:07.571 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:07.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:07.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:07.572 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:08.470 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:08.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:08.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:08.471 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:09.891 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:09.891 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:09.892 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=20, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=20, TimeoutTracker(timeout=30000, elapsed=10797), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158)))) after lookup
07:00:09.893 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:09.894 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:09.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:09.894 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:11.431 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:11.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:11.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:11.432 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:15.170 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:15.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:15.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:15.172 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:17.311 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:17.311 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:17.312 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=24, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=24, TimeoutTracker(timeout=30000, elapsed=18217), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917)))) after lookup
07:00:17.313 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:17.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:17.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:17.314 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:20.370 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:20.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:20.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:20.371 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:22.990 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:22.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:22.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:22.991 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:26.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:26.391 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:26.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=28, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=28, TimeoutTracker(timeout=30000, elapsed=27297), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814)))) after lookup
07:00:26.393 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:26.394 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:26.394 [DEBUG - kudu-nio-0] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:26.394 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:26.850 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:26.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:26.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:26.851 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:28.630 [DEBUG - pool-1024-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=getTableKeyRanges, tablet=null, attempt=0, TimeoutTracker(timeout=30000, elapsed=30001), Traces: [0ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [12ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: OK, deferred=null) with no deferred attached!
07:00:28.950 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] connecting to peer
07:00:28.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:28.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:28.951 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
07:00:28.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
07:00:28.952 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
	at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
	at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
	at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
	at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:00:28.958 [ERROR - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:94) Exception in task 2.0 in stage 0.0 (TID 2)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144) ~[main/:?]
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:00:29.001 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 3.0 in stage 0.0 (TID 3) (dist-test-slave-str9.c.gcp-upstream.internal, executor driver, partition 3, ANY, 5297 bytes) taskResourceAssignments Map()
07:00:29.001 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (Logging.scala:57) Running task 3.0 in stage 0.0 (TID 3)
07:00:29.003 [WARN - task-result-getter-2] (Logging.scala:69) Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

07:00:29.005 [ERROR - task-result-getter-2] (Logging.scala:73) Task 2 in stage 0.0 failed 1 times; aborting job
07:00:29.008 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
07:00:29.008 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
07:00:29.025 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Executor is trying to kill task 3.0 in stage 0.0 (TID 3), reason: Stage cancelled
07:00:29.026 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
07:00:29.027 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (save at KuduBackup.scala:97) failed in 30.368 s due to Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
07:00:29.027 [DEBUG - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:00:29.028 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Job 0 failed: save at KuduBackup.scala:97, took 30.371152 s
07:00:29.030 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 8dac1bb306d340ce8d8b0079e1725d62
07:00:29.031 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:00:29.031 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:00:29.030 [ERROR - ForkJoinPool-240-worker-1] (Logging.scala:94) Aborting job 3f9adbbf-59dc-457b-8487-7e71c99f6129.
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
	at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
07:00:29.031 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:00:29.032 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:00:29.032 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
	at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
	at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
	at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
	at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
	at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
	at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
	at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.Try$.apply(Try.scala:213)
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
	at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
	at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
	at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
	at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
	at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
	at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
	at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
	at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)

07:00:29.032 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:00:29.034 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 30532dbf58484952b9456afd5e75113f for table 'split-size-table' with partition [<start>, <end>)
07:00:29.034 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=30532dbf58484952b9456afd5e75113f@[0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)[L],686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471),cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)]}]
07:00:29.034 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [0ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, deferred=null)) after lookup
07:00:29.034 [DEBUG - kudu-nio-1] (Connection.java:748) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] connecting to peer
07:00:29.035 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:29.035 [DEBUG - kudu-nio-1] (Connection.java:709) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] cleaning up while in state CONNECTING due to: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:29.035 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695
07:00:29.035 [ERROR - ForkJoinPool-240-worker-1] (KuduBackup.scala:154) Failed to back up table split-size-table
org.apache.spark.SparkException: Job aborted.
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
	at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
	... 64 more
org.apache.spark.SparkException: Job aborted.
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231)
	at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
	at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
	at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
	at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
	at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
	at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
	at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
	at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.Try$.apply(Try.scala:213)
	at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
	at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
	at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
	at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
	at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
	at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
	at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
	at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
	at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
	at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
	at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
	at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
	at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
	at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
	at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
	at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
	at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
	at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-str9.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
	at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, KuduRpc(method=Scan, tablet=30532dbf58484952b9456afd5e75113f, attempt=31, TimeoutTracker(timeout=30000, elapsed=29856), Traces: [0ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [172ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: connection closed, [172ms] delaying RPC due to: Network error: connection closed, [177ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [355ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: connection closed, [355ms] delaying RPC due to: Network error: connection closed, [376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [541ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: connection closed, [541ms] delaying RPC due to: Network error: connection closed, [556ms] refreshing cache from master, [556ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [557ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [558ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [560ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [560ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [576ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [577ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [577ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [676ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [677ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [678ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [679ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [679ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [758ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [758ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [1596ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [1598ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [1598ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [2896ms] refreshing cache from master, [2896ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [2897ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [2898ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [2899ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [2900ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [5276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [5278ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [5278ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [7636ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [7637ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7637ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [7956ms] refreshing cache from master, [7956ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [7957ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [7958ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [7960ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [7960ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [8476ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [8478ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [8478ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [9376ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [9377ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [9377ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [10796ms] refreshing cache from master, [10796ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [10797ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [10799ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [10800ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [10800ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [12336ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [12338ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [12338ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [16076ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [16078ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [16078ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [18216ms] refreshing cache from master, [18216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [18217ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [18219ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [18220ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [18220ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [21276ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [21277ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [21277ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [23896ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [23897ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [23897ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [27296ms] refreshing cache from master, [27296ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.94.188:34325, [27297ms] Sub RPC GetTableLocations: received response from server master-127.10.94.188:34325: OK, [27299ms] sending RPC to server cdde07f1fbaf496eb82c215b5e7a0215, [27300ms] received response from server cdde07f1fbaf496eb82c215b5e7a0215: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27300ms] delaying RPC due to: Network error: Failed to connect to peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695): Connection refused: /127.10.94.131:46695, [27756ms] sending RPC to server 686537b603c94bc3a797ce5808b5d5bf, [27757ms] received response from server 686537b603c94bc3a797ce5808b5d5bf: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [27757ms] delaying RPC due to: Network error: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471, [29856ms] sending RPC to server 0154f5115a5d4ca9b2e2cced3e3eb545, [29857ms] received response from server 0154f5115a5d4ca9b2e2cced3e3eb545: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, [29857ms] delaying RPC due to: Network error: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247, deferred=Deferred@525476060(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@41576868 after retry RPC@567969559) -> (continuation of Deferred@1707282479 after retry RPC@928577207) -> (continuation of Deferred@134098228 after retry RPC@161587744) -> (continuation of Deferred@1331114029 after retry RPC@1931626158) -> (continuation of Deferred@1618209190 after retry RPC@2022428917) -> (continuation of Deferred@1645138440 after retry RPC@376523814) -> (continuation of Deferred@1615256528 after retry RPC@719839554))))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
	at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
	... 15 more
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
		at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
		at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
		at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
		at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247): Connection refused: /127.10.94.129:36247
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.94.129:36247
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
	... 64 more

07:00:29.039 [ERROR - ForkJoinPool-240-worker-1] (KuduBackup.scala:162) Failed to back up table split-size-table: Look back in the logs for the full exception. Error: Failure(org.apache.spark.SparkException: Job aborted.)
07:00:29.043 [INFO - dispatcher-event-loop-1] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
07:00:29.048 [INFO - main] (Logging.scala:57) MemoryStore cleared
07:00:29.048 [INFO - main] (Logging.scala:57) BlockManager stopped
07:00:29.048 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
07:00:29.048 [INFO - dispatcher-event-loop-0] (Logging.scala:57) OutputCommitCoordinator stopped!
07:00:29.050 [DEBUG - pool-1024-thread-1] (Connection.java:748) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] connecting to peer
07:00:29.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:29.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] cleaning up while in state CONNECTING due to: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:29.051 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471) for tablet 30532dbf58484952b9456afd5e75113f: Failed to connect to peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471): Connection refused: /127.10.94.130:33471
07:00:29.052 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
07:00:29.052 [DEBUG - main] (Connection.java:575) [peer cdde07f1fbaf496eb82c215b5e7a0215(127.10.94.131:46695)] disconnecting while in state TERMINATED
07:00:29.052 [DEBUG - main] (Connection.java:575) [peer 0154f5115a5d4ca9b2e2cced3e3eb545(127.10.94.129:36247)] disconnecting while in state TERMINATED
07:00:29.052 [DEBUG - main] (Connection.java:575) [peer master-127.10.94.188:34325(127.10.94.188:34325)] disconnecting while in state READY
07:00:29.052 [DEBUG - main] (Connection.java:575) [peer 686537b603c94bc3a797ce5808b5d5bf(127.10.94.130:33471)] disconnecting while in state TERMINATED
07:00:29.053 [DEBUG - main] (Connection.java:575) [peer master-127.10.94.189:37271(127.10.94.189:37271)] disconnecting while in state READY
07:00:29.053 [DEBUG - main] (Connection.java:575) [peer master-127.10.94.190:37303(127.10.94.190:37303)] disconnecting while in state READY
07:00:29.053 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.94.188:34325(127.10.94.188:34325)] handling channelInactive
07:00:29.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.94.188:34325(127.10.94.188:34325)] cleaning up while in state READY due to: connection closed
07:00:29.053 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:00:29.054 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.94.189:37271(127.10.94.189:37271)] handling channelInactive
07:00:29.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.94.189:37271(127.10.94.189:37271)] cleaning up while in state READY due to: connection closed
07:00:29.054 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.94.190:37303(127.10.94.190:37303)] handling channelInactive
07:00:29.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.94.190:37303(127.10.94.190:37303)] cleaning up while in state READY due to: connection closed
07:00:29.054 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 30 thread-local buffer(s) from thread: kudu-nio-0
07:00:29.055 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 36 thread-local buffer(s) from thread: kudu-nio-1
07:00:29.063 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:29.064023 10618 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklQverR/build/release/bin/kudu with pid 10629
07:00:29.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:29.069344 10618 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklQverR/build/release/bin/kudu with pid 10700
07:00:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:29.074051 10618 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasklQverR/build/release/bin/kudu with pid 10771
07:00:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:00:29Z chronyd exiting
07:00:29.125 [ERROR - main] (RetryRule.java:206) org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes: failed attempt 1
java.lang.AssertionError: null
	at org.junit.Assert.fail(Assert.java:87) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.Assert.assertTrue(Assert.java:53) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414) ~[test/:?]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
	at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
	at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)