06:58:42.132 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:58:42.132 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task31bC9m/test-tmp
06:58:42.132 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
06:58:42.132 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task31bC9m/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
06:58:42.136 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984"
extra_tserver_flags: "--flush_threshold_mb=1"
extra_tserver_flags: "--flush_threshold_secs=1"
extra_tserver_flags: "--enable_rowset_compaction=false"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:58:42.219 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:58:42.219 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:58:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.220242 25101 mini_oidc.cc:152] Starting JWKS server
06:58:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.221112 25101 webserver.cc:460] Webserver started at http://127.0.0.1:44005/ using document root <none> and password file <none>
06:58:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.221189 25101 mini_oidc.cc:189] Starting OIDC Discovery server
06:58:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.221329 25101 webserver.cc:460] Webserver started at http://127.0.0.1:34157/ using document root <none> and password file <none>
06:58:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:58:42Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:58:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:58:42Z Disabled control of system clock
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.233326 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/logs
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data/info.pb
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.126:46597
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.126
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json with env {}
06:58:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.315013 25112 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.315222 25112 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.315245 25112 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316691 25112 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316744 25112 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316758 25112 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316774 25112 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316794 25112 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:58:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.316814 25112 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44005/jwks.json
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.318332 25112 master_runner.cc:386] Master server non-default flags:
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.126:46597
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data/info.pb
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.126
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25112
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/logs
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.318547 25112 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.318759 25112 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:42.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.323412 25120 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.324097 25117 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.324645 25118 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.325474 25112 server_base.cc:1034] running on GCE node
06:58:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.325726 25112 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:42.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.325954 25112 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:42.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.327102 25112 hybrid_clock.cc:648] HybridClock initialized: now 1714114722327083 us; error 42 us; skew 500 ppm
06:58:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.328493 25112 webserver.cc:460] Webserver started at http://127.24.131.126:44441/ using document root <none> and password file <none>
06:58:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.328663 25112 fs_manager.cc:359] Metadata directory not provided
06:58:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.328701 25112 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.328785 25112 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.329699 25112 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data/instance:
06:58:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad569656bca0471bb66afda61519c288"
06:58:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.329964 25112 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal/instance:
06:58:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad569656bca0471bb66afda61519c288"
06:58:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.331193 25112 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.002s sys 0.001s
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.331948 25126 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.332183 25112 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.001s sys 0.000s
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.332269 25112 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad569656bca0471bb66afda61519c288"
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.332327 25112 fs_report.cc:383] FS layout report
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data/data
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.342208 25112 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.342449 25112 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.342535 25112 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.346269 25112 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.126:46597
06:58:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.346302 25178 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.126:46597 every 8 connection(s)
06:58:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.346580 25112 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/data/info.pb
06:58:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.347151 25179 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.347524 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25112
06:58:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.347604 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-0/wal/instance
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.348489 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/logs
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data/info.pb
06:58:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.125:32773
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.125
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json with env {}
06:58:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.349124 25179 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.354776 25179 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.355201 25130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.131.125:32773: connect: Connection refused (error 111)
06:58:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.355655 25179 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.125:32773: Network error: Client connection negotiation failed: client connection to 127.24.131.125:32773: connect: Connection refused (error 111)
06:58:42.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.404930 25179 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } attempt: 1
06:58:42.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.405827 25179 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.125:32773: Network error: Client connection negotiation failed: client connection to 127.24.131.125:32773: connect: Connection refused (error 111)
06:58:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.432464 25181 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.432682 25181 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.432705 25181 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434128 25181 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434175 25181 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434190 25181 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434203 25181 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434216 25181 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:58:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.434229 25181 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44005/jwks.json
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.435863 25181 master_runner.cc:386] Master server non-default flags:
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.125:32773
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data/info.pb
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.125
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25181
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/logs
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.436095 25181 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.436314 25181 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.441038 25190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.441485 25192 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.442175 25189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.442684 25181 server_base.cc:1034] running on GCE node
06:58:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.442965 25181 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.443248 25181 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:42.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.444391 25181 hybrid_clock.cc:648] HybridClock initialized: now 1714114722444374 us; error 33 us; skew 500 ppm
06:58:42.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.445793 25181 webserver.cc:460] Webserver started at http://127.24.131.125:37769/ using document root <none> and password file <none>
06:58:42.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.446017 25181 fs_manager.cc:359] Metadata directory not provided
06:58:42.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.446072 25181 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:42.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.446195 25181 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.447208 25181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data/instance:
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74efbb5c221b4014a098f214d6cb4027"
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.447556 25181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal/instance:
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74efbb5c221b4014a098f214d6cb4027"
06:58:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.448876 25181 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.000s sys 0.001s
06:58:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.449576 25198 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.449782 25181 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s
06:58:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.449872 25181 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74efbb5c221b4014a098f214d6cb4027"
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.449954 25181 fs_report.cc:383] FS layout report
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data/data
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.469096 25181 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.469444 25181 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.469583 25181 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.473450 25181 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.125:32773
06:58:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.473495 25250 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.125:32773 every 8 connection(s)
06:58:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.473845 25181 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/data/info.pb
06:58:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.473991 25179 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } attempt: 2
06:58:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.474579 25251 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:42.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.477030 25251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.479691 25179 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.480525 25179 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.481086 25251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.483268 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25181
06:58:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.483384 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-1/wal/instance
06:58:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.483862 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/logs
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data/info.pb
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.124:42491
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.124
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json with env {}
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.483837 25251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.484329 25199 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.484830 25251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.523767 25179 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 1
06:58:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.524657 25179 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.534072 25251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 1
06:58:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.534988 25251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.569849 25256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.570067 25256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.570103 25256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571657 25256 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:58:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571743 25256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571770 25256 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571792 25256 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571816 25256 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.571835 25256 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44005/jwks.json
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.571859 25179 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 2
06:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.572634 25179 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.573769 25256 master_runner.cc:386] Master server non-default flags:
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.124:42491
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44005/jwks.json
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data/info.pb
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.124
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25256
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/logs
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.574050 25256 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.574362 25256 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.579667 25261 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.580353 25262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.580893 25264 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.581658 25256 server_base.cc:1034] running on GCE node
06:58:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.581920 25256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.582207 25256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:42.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.583353 25256 hybrid_clock.cc:648] HybridClock initialized: now 1714114722583335 us; error 35 us; skew 500 ppm
06:58:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.584827 25256 webserver.cc:460] Webserver started at http://127.24.131.124:46745/ using document root <none> and password file <none>
06:58:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.585048 25256 fs_manager.cc:359] Metadata directory not provided
06:58:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.585104 25256 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.585228 25256 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.586280 25256 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data/instance:
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a44e998012d4bdd9a1ca8db4ae66364"
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.586620 25256 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal/instance:
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a44e998012d4bdd9a1ca8db4ae66364"
06:58:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.587863 25256 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.002s sys 0.001s
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.588518 25270 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.588667 25256 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.588737 25256 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a44e998012d4bdd9a1ca8db4ae66364"
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.588784 25256 fs_report.cc:383] FS layout report
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data/data
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.603205 25251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 2
06:58:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.604118 25251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.131.124:42491: Network error: Client connection negotiation failed: client connection to 127.24.131.124:42491: connect: Connection refused (error 111)
06:58:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.607547 25256 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.607858 25256 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.607954 25256 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.611415 25256 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.124:42491
06:58:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.611469 25322 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.124:42491 every 8 connection(s)
06:58:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.611837 25256 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/data/info.pb
06:58:42.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.612516 25323 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:42.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.614161 25323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.618687 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25256
06:58:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.618784 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/master-2/wal/instance
06:58:42.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.618750 25323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.621924 25323 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } has no permanent_uuid. Determining permanent_uuid...
06:58:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.626636 25323 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: Bootstrap starting.
06:58:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.627274 25323 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: Neither blocks nor log segments found. Creating new log.
06:58:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.627588 25323 log.cc:824] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: Log is configured to *not* fsync() on all Append() calls
06:58:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.628307 25323 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: No bootstrap required, opened a new log
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630210 25323 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630323 25323 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630354 25323 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a44e998012d4bdd9a1ca8db4ae66364, State: Initialized, Role: FOLLOWER
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630458 25323 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630759 25323 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: configured and running, proceeding with master startup.
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630734 25331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.630808 25331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.632506 25342 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:58:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.632575 25342 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:58:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.632876 25256 master_runner.cc:186] Error getting master registration for 127.24.131.126:46597: OK, instance_id { permanent_uuid: "ad569656bca0471bb66afda61519c288" instance_seqno: 1714114722345230 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:58:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.634354 25256 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:58:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.679780 25179 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 3
06:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.684511 25179 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: Bootstrap starting.
06:58:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.685271 25179 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: Neither blocks nor log segments found. Creating new log.
06:58:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.685567 25179 log.cc:824] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: Log is configured to *not* fsync() on all Append() calls
06:58:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.686511 25179 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: No bootstrap required, opened a new log
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689013 25179 raft_consensus.cc:348] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689170 25179 raft_consensus.cc:374] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689200 25179 raft_consensus.cc:729] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad569656bca0471bb66afda61519c288, State: Initialized, Role: FOLLOWER
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689327 25179 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689677 25179 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: configured and running, proceeding with master startup.
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689666 25345 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.689788 25345 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.692020 25356 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:58:42.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.692324 25356 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:58:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.692975 25112 master_runner.cc:186] Error getting master registration for 127.24.131.125:32773: OK, instance_id { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" instance_seqno: 1714114722472387 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:58:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.694447 25112 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:58:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.695266 25251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } attempt: 3
06:58:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.699138 25251 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: Bootstrap starting.
06:58:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.699901 25251 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: Neither blocks nor log segments found. Creating new log.
06:58:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.700412 25251 log.cc:824] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: Log is configured to *not* fsync() on all Append() calls
06:58:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.701083 25251 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: No bootstrap required, opened a new log
06:58:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702343 25251 raft_consensus.cc:348] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702441 25251 raft_consensus.cc:374] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702469 25251 raft_consensus.cc:729] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74efbb5c221b4014a098f214d6cb4027, State: Initialized, Role: FOLLOWER
06:58:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702592 25251 consensus_queue.cc:260] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702947 25251 sys_catalog.cc:564] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: configured and running, proceeding with master startup.
06:58:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.702910 25359 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.703168 25359 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.704830 25370 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:58:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.704993 25370 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:58:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.706918 25181 master_runner.cc:263] No leader master found from master 74efbb5c221b4014a098f214d6cb4027
06:58:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.706962 25181 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:58:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.723044 25204 tablet.cc:2369] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:58:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.745469 25359 raft_consensus.cc:482] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:58:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.745604 25359 raft_consensus.cc:504] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.745874 25359 leader_election.cc:288] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ad569656bca0471bb66afda61519c288 (127.24.131.126:46597), 2a44e998012d4bdd9a1ca8db4ae66364 (127.24.131.124:42491)
06:58:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.746212 25297 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "74efbb5c221b4014a098f214d6cb4027" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" is_pre_election: true
06:58:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.746346 25297 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74efbb5c221b4014a098f214d6cb4027 in term 0.
06:58:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.746541 25199 leader_election.cc:302] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a44e998012d4bdd9a1ca8db4ae66364, 74efbb5c221b4014a098f214d6cb4027; no voters:
06:58:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.747341 25153 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "74efbb5c221b4014a098f214d6cb4027" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad569656bca0471bb66afda61519c288" is_pre_election: true
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.747494 25153 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74efbb5c221b4014a098f214d6cb4027 in term 0.
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.747774 25359 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.747846 25359 raft_consensus.cc:482] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.747872 25359 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 0 FOLLOWER]: Advancing to term 1
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.748834 25359 raft_consensus.cc:504] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.748975 25359 leader_election.cc:288] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [CANDIDATE]: Term 1 election: Requested vote from peers ad569656bca0471bb66afda61519c288 (127.24.131.126:46597), 2a44e998012d4bdd9a1ca8db4ae66364 (127.24.131.124:42491)
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.749186 25297 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "74efbb5c221b4014a098f214d6cb4027" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a44e998012d4bdd9a1ca8db4ae66364"
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.749265 25297 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 0 FOLLOWER]: Advancing to term 1
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.750090 25297 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74efbb5c221b4014a098f214d6cb4027 in term 1.
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.750272 25199 leader_election.cc:302] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a44e998012d4bdd9a1ca8db4ae66364, 74efbb5c221b4014a098f214d6cb4027; no voters:
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.750571 25153 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "74efbb5c221b4014a098f214d6cb4027" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad569656bca0471bb66afda61519c288"
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.750658 25153 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 0 FOLLOWER]: Advancing to term 1
06:58:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.751554 25153 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74efbb5c221b4014a098f214d6cb4027 in term 1.
06:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.752528 25359 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 1 FOLLOWER]: Leader election won for term 1
06:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.752718 25359 raft_consensus.cc:686] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [term 1 LEADER]: Becoming Leader. State: Replica: 74efbb5c221b4014a098f214d6cb4027, State: Running, Role: LEADER
06:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.752815 25359 consensus_queue.cc:237] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } }
06:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.753222 25359 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 74efbb5c221b4014a098f214d6cb4027. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.753299 25359 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: This master's current role is: LEADER
06:58:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.754611 25374 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:58:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.754957 25374 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:58:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.758019 25297 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [term 1 FOLLOWER]: Refusing update from remote peer 74efbb5c221b4014a098f214d6cb4027: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:58:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.758496 25372 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:58:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.760061 25331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 74efbb5c221b4014a098f214d6cb4027. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.760166 25331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.760730 25153 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [term 1 FOLLOWER]: Refusing update from remote peer 74efbb5c221b4014a098f214d6cb4027: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:58:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.761106 25372 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:58:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.762364 25345 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 74efbb5c221b4014a098f214d6cb4027. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.762449 25345 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.764775 25372 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.764863 25372 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: This master's current role is: LEADER
06:58:42.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.765516 25331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.765599 25331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.766316 25375 mvcc.cc:205] Tried to move back new op lower bound from 7021013904409075712 to 7021013904396079104. Current Snapshot: MvccSnapshot[applied={T|T < 7021013904409075712}]
06:58:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.766508 25376 mvcc.cc:205] Tried to move back new op lower bound from 7021013904409075712 to 7021013904396079104. Current Snapshot: MvccSnapshot[applied={T|T < 7021013904409075712}]
06:58:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.766988 25359 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.767064 25359 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027 [sys.catalog]: This master's current role is: LEADER
06:58:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.767493 25345 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "74efbb5c221b4014a098f214d6cb4027" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad569656bca0471bb66afda61519c288" member_type: VOTER last_known_addr { host: "127.24.131.126" port: 46597 } } peers { permanent_uuid: "74efbb5c221b4014a098f214d6cb4027" member_type: VOTER last_known_addr { host: "127.24.131.125" port: 32773 } } peers { permanent_uuid: "2a44e998012d4bdd9a1ca8db4ae66364" member_type: VOTER last_known_addr { host: "127.24.131.124" port: 42491 } } }
06:58:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.767575 25345 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288 [sys.catalog]: This master's current role is: FOLLOWER
06:58:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.768152 25374 catalog_manager.cc:1347] Generated new cluster ID: 421afb2b5e1d4349a7c4cd77eafb3c90
06:58:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.768196 25374 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:58:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.768051 25378 mvcc.cc:205] Tried to move back new op lower bound from 7021013904409075712 to 7021013904396079104. Current Snapshot: MvccSnapshot[applied={T|T < 7021013904409075712}]
06:58:42.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.777210 25374 catalog_manager.cc:1370] Generated new certificate authority record
06:58:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.778172 25374 catalog_manager.cc:1504] Loading token signing keys...
06:58:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.791812 25374 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 74efbb5c221b4014a098f214d6cb4027: Generated new TSK 0
06:58:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.792397 25374 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.797497 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/logs
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data/info.pb
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.65:0
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.65
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.65
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:58:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.901762 25383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.901969 25383 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.902001 25383 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:58:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.902024 25383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.903510 25383 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.903574 25383 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:58:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.903595 25383 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:58:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.903622 25383 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.131.65
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.905241 25383 tablet_server_runner.cc:78] Tablet server non-default flags:
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.65:0
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data/info.pb
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.65
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25383
06:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.65
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/logs
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.905478 25383 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.905754 25383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:42.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.910925 25388 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.911334 25389 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:42.912159 25391 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:42.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.912788 25383 server_base.cc:1034] running on GCE node
06:58:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.913007 25383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.913260 25383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.914419 25383 hybrid_clock.cc:648] HybridClock initialized: now 1714114722914400 us; error 39 us; skew 500 ppm
06:58:42.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.915956 25383 webserver.cc:460] Webserver started at http://127.24.131.65:39671/ using document root <none> and password file <none>
06:58:42.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.916198 25383 fs_manager.cc:359] Metadata directory not provided
06:58:42.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.916253 25383 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:42.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.916374 25383 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.917436 25383 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data/instance:
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe51ffe4ac064164bf25f1c5653c94d0"
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.917785 25383 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal/instance:
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe51ffe4ac064164bf25f1c5653c94d0"
06:58:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.919016 25383 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.003s sys 0.000s
06:58:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.919728 25397 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.920001 25383 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.001s sys 0.000s
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.920087 25383 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe51ffe4ac064164bf25f1c5653c94d0"
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:42 on dist-test-slave-s24k"
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.920163 25383 fs_report.cc:383] FS layout report
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data/data
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.940371 25383 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.940681 25383 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.940811 25383 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.941066 25383 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:58:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.941448 25383 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:58:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.941488 25383 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:58:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.941524 25383 ts_tablet_manager.cc:590] Registered 0 tablets
06:58:42.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.941540 25383 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:58:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.949015 25383 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.65:37653
06:58:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.949385 25383 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/data/info.pb
06:58:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.950492 25510 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.65:37653 every 8 connection(s)
06:58:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.955940 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25383
06:58:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.956063 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-0/wal/instance
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.956885 25512 heartbeater.cc:347] Connected to a master server at 127.24.131.124:42491
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.956985 25512 heartbeater.cc:464] Registering TS with master...
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.957204 25512 heartbeater.cc:511] Master 127.24.131.124:42491 requested a full tablet report, sending...
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.957289 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/logs
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data/info.pb
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.66:0
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.66
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.66
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.957674 25286 ts_manager.cc:194] Registered new tserver with Master: fe51ffe4ac064164bf25f1c5653c94d0 (127.24.131.65:37653)
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.958474 25511 heartbeater.cc:347] Connected to a master server at 127.24.131.126:46597
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.958534 25511 heartbeater.cc:464] Registering TS with master...
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.958667 25511 heartbeater.cc:511] Master 127.24.131.126:46597 requested a full tablet report, sending...
06:58:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.959039 25143 ts_manager.cc:194] Registered new tserver with Master: fe51ffe4ac064164bf25f1c5653c94d0 (127.24.131.65:37653)
06:58:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.960865 25513 heartbeater.cc:347] Connected to a master server at 127.24.131.125:32773
06:58:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.960928 25513 heartbeater.cc:464] Registering TS with master...
06:58:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.961007 25513 heartbeater.cc:511] Master 127.24.131.125:32773 requested a full tablet report, sending...
06:58:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.961326 25215 ts_manager.cc:194] Registered new tserver with Master: fe51ffe4ac064164bf25f1c5653c94d0 (127.24.131.65:37653)
06:58:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:42.962456 25215 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.131.65:34831
06:58:43.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.044751 25518 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:43.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.044968 25518 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:43.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.044991 25518 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:58:43.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.045008 25518 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:43.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.046435 25518 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:43.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.046485 25518 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:58:43.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.046499 25518 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:58:43.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.046519 25518 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.131.66
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.048071 25518 tablet_server_runner.cc:78] Tablet server non-default flags:
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.66:0
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data/info.pb
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.66
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25518
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.66
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/logs
06:58:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.048295 25518 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:43.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.048519 25518 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.053956 25524 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.054255 25526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.054960 25523 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.055516 25518 server_base.cc:1034] running on GCE node
06:58:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.055676 25518 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:43.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.055915 25518 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.057060 25518 hybrid_clock.cc:648] HybridClock initialized: now 1714114723057034 us; error 44 us; skew 500 ppm
06:58:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.058434 25518 webserver.cc:460] Webserver started at http://127.24.131.66:40843/ using document root <none> and password file <none>
06:58:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.058647 25518 fs_manager.cc:359] Metadata directory not provided
06:58:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.058703 25518 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.058828 25518 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.059870 25518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data/instance:
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "436da135bace402ba6dede0b5d9f6ab5"
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.060225 25518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal/instance:
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "436da135bace402ba6dede0b5d9f6ab5"
06:58:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.061705 25518 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.000s sys 0.002s
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.062377 25532 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.062563 25518 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.062640 25518 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "436da135bace402ba6dede0b5d9f6ab5"
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.062695 25518 fs_report.cc:383] FS layout report
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal
06:58:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data/data
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.085343 25518 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.085915 25518 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.086119 25518 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.086566 25518 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:58:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.087158 25518 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:58:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.087249 25518 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:58:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.087337 25518 ts_tablet_manager.cc:590] Registered 0 tablets
06:58:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.087388 25518 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:58:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.095393 25518 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.66:35223
06:58:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.095456 25645 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.66:35223 every 8 connection(s)
06:58:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.095826 25518 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/data/info.pb
06:58:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101307 25646 heartbeater.cc:347] Connected to a master server at 127.24.131.126:46597
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101308 25647 heartbeater.cc:347] Connected to a master server at 127.24.131.124:42491
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101459 25646 heartbeater.cc:464] Registering TS with master...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101461 25647 heartbeater.cc:464] Registering TS with master...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101540 25648 heartbeater.cc:347] Connected to a master server at 127.24.131.125:32773
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101580 25648 heartbeater.cc:464] Registering TS with master...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101665 25646 heartbeater.cc:511] Master 127.24.131.126:46597 requested a full tablet report, sending...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101666 25648 heartbeater.cc:511] Master 127.24.131.125:32773 requested a full tablet report, sending...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.101666 25647 heartbeater.cc:511] Master 127.24.131.124:42491 requested a full tablet report, sending...
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.102054 25215 ts_manager.cc:194] Registered new tserver with Master: 436da135bace402ba6dede0b5d9f6ab5 (127.24.131.66:35223)
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.102454 25143 ts_manager.cc:194] Registered new tserver with Master: 436da135bace402ba6dede0b5d9f6ab5 (127.24.131.66:35223)
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.102466 25215 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.131.66:34747
06:58:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.102854 25286 ts_manager.cc:194] Registered new tserver with Master: 436da135bace402ba6dede0b5d9f6ab5 (127.24.131.66:35223)
06:58:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.103857 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25518
06:58:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.103951 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-1/wal/instance
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.105294 25101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task31bC9m/build/release/bin/kudu
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/logs
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data/info.pb
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:58:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.67:0
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.67
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.67
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
06:58:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.193117 25653 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:58:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.193311 25653 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:58:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.193331 25653 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
06:58:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.193346 25653 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:58:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.194645 25653 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:58:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.194693 25653 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
06:58:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.194712 25653 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
06:58:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.194728 25653 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.131.67
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.196194 25653 tablet_server_runner.cc:78] Tablet server non-default flags:
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.131.84:32787
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.131.67:0
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data/info.pb
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.131.67
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.131.126:46597,127.24.131.125:32773,127.24.131.124:42491
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.25653
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.131.67
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/logs
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:44 UTC on 1a15035a6830
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 250
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.196388 25653 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:58:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.196594 25653 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:58:43.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.201653 25659 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.202404 25658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.203027 25661 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:58:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.203763 25653 server_base.cc:1034] running on GCE node
06:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.203951 25653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.204192 25653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.205349 25653 hybrid_clock.cc:648] HybridClock initialized: now 1714114723205292 us; error 79 us; skew 500 ppm
06:58:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.206789 25653 webserver.cc:460] Webserver started at http://127.24.131.67:38125/ using document root <none> and password file <none>
06:58:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.206967 25653 fs_manager.cc:359] Metadata directory not provided
06:58:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.207008 25653 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:58:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.207096 25653 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.208141 25653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data/instance:
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0d6561d1e1145b78f710614a3187cb4"
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.208437 25653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal/instance:
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0d6561d1e1145b78f710614a3187cb4"
06:58:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.209620 25653 fs_manager.cc:691] Time spent creating directory manager: real 0.001s user 0.002s sys 0.000s
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.210278 25667 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.210443 25653 fs_manager.cc:725] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.210513 25653 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data,/tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0d6561d1e1145b78f710614a3187cb4"
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:58:43 on dist-test-slave-s24k"
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.210563 25653 fs_report.cc:383] FS layout report
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data/data
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:58:43.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:58:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.219516 25653 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:58:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.219806 25653 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:58:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.219892 25653 kserver.cc:163] Server-wide thread pool size limit: 3276
06:58:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.220072 25653 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:58:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.220360 25653 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:58:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.220391 25653 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:58:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.220415 25653 ts_tablet_manager.cc:590] Registered 0 tablets
06:58:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.220427 25653 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:58:43.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.226671 25653 rpc_server.cc:307] RPC server started. Bound to: 127.24.131.67:36017
06:58:43.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.226732 25780 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.131.67:36017 every 8 connection(s)
06:58:43.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.227058 25653 server_base.cc:1166] Dumped server information to /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/data/info.pb
06:58:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.230957 25101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task31bC9m/build/release/bin/kudu as pid 25653
06:58:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.231053 25101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task31bC9m/test-tmp/mini-kudu-cluster5987860316724503984/ts-2/wal/instance
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233084 25782 heartbeater.cc:347] Connected to a master server at 127.24.131.124:42491
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233201 25782 heartbeater.cc:464] Registering TS with master...
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233441 25782 heartbeater.cc:511] Master 127.24.131.124:42491 requested a full tablet report, sending...
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233466 25781 heartbeater.cc:347] Connected to a master server at 127.24.131.126:46597
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233525 25781 heartbeater.cc:464] Registering TS with master...
06:58:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233628 25781 heartbeater.cc:511] Master 127.24.131.126:46597 requested a full tablet report, sending...
06:58:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233886 25286 ts_manager.cc:194] Registered new tserver with Master: f0d6561d1e1145b78f710614a3187cb4 (127.24.131.67:36017)
06:58:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.233928 25143 ts_manager.cc:194] Registered new tserver with Master: f0d6561d1e1145b78f710614a3187cb4 (127.24.131.67:36017)
06:58:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.236500 25783 heartbeater.cc:347] Connected to a master server at 127.24.131.125:32773
06:58:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.236565 25783 heartbeater.cc:464] Registering TS with master...
06:58:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.236639 25783 heartbeater.cc:511] Master 127.24.131.125:32773 requested a full tablet report, sending...
06:58:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.236955 25215 ts_manager.cc:194] Registered new tserver with Master: f0d6561d1e1145b78f710614a3187cb4 (127.24.131.67:36017)
06:58:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.237474 25215 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.131.67:58139
06:58:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.246409 25101 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:58:43.247 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:58:43.247 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:58:43.247 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.131.126"
port: 46597
}
bound_http_address {
host: "127.24.131.126"
port: 44441
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.131.125"
port: 32773
}
bound_http_address {
host: "127.24.131.125"
port: 37769
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.131.124"
port: 42491
}
bound_http_address {
host: "127.24.131.124"
port: 46745
}
}
}
06:58:43.248 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:58:43.248 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.131.65"
port: 37653
}
bound_http_address {
host: "127.24.131.65"
port: 39671
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.131.66"
port: 35223
}
bound_http_address {
host: "127.24.131.66"
port: 40843
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.131.67"
port: 36017
}
bound_http_address {
host: "127.24.131.67"
port: 38125
}
}
}
06:58:43.249 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:58:43.252 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:58:43.253 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
06:58:43.254 [INFO - main] (Logging.scala:57) ==============================================================
06:58:43.254 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
06:58:43.254 [INFO - main] (Logging.scala:57) ==============================================================
06:58:43.254 [INFO - main] (Logging.scala:57) Submitted application: test
06:58:43.255 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
06:58:43.255 [INFO - main] (Logging.scala:57) Limiting resource is cpu
06:58:43.256 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
06:58:43.257 [INFO - main] (Logging.scala:57) Changing view acls to: slave
06:58:43.257 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
06:58:43.257 [INFO - main] (Logging.scala:57) Changing view acls groups to:
06:58:43.257 [INFO - main] (Logging.scala:57) Changing modify acls groups to:
06:58:43.257 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(slave); groups with view permissions: Set(); users with modify permissions: Set(slave); groups with modify permissions: Set()
06:58:43.279 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 44185.
06:58:43.281 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
06:58:43.282 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
06:58:43.283 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
06:58:43.283 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
06:58:43.283 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
06:58:43.284 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-d033fd33-d38c-42c6-9402-b534deb18a0a
06:58:43.284 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
06:58:43.286 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
06:58:43.309 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-s24k.c.gcp-upstream.internal
06:58:43.313 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 41561.
06:58:43.313 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-s24k.c.gcp-upstream.internal:41561
06:58:43.313 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
06:58:43.313 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-s24k.c.gcp-upstream.internal, 41561, None)
06:58:43.314 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-s24k.c.gcp-upstream.internal:41561 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-s24k.c.gcp-upstream.internal, 41561, None)
06:58:43.315 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-s24k.c.gcp-upstream.internal, 41561, None)
06:58:43.316 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-s24k.c.gcp-upstream.internal, 41561, None)
06:58:43.325 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:58:43.338 [DEBUG - main] (Connection.java:748) [peer master-127.24.131.126:46597(127.24.131.126:46597)] connecting to peer
06:58:43.340 [DEBUG - main] (Connection.java:748) [peer master-127.24.131.125:32773(127.24.131.125:32773)] connecting to peer
06:58:43.341 [DEBUG - main] (Connection.java:748) [peer master-127.24.131.124:42491(127.24.131.124:42491)] connecting to peer
06:58:43.342 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.131.126:46597(127.24.131.126:46597)] Successfully connected to peer
06:58:43.349 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.24.131.126
06:58:43.352 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.131.125:32773(127.24.131.125:32773)] Successfully connected to peer
06:58:43.354 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.131.124:42491(127.24.131.124:42491)] Successfully connected to peer
06:58:43.355 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.24.131.125
06:58:43.360 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:58:43.363 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x43cc3139, L:/127.0.0.1:44970 - R:/127.24.131.126:46597] using SASL/PLAIN
06:58:43.364 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.24.131.124
06:58:43.368 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:58:43.371 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7e54473a, L:/127.0.0.1:46598 - R:/127.24.131.125:32773] using SASL/PLAIN
06:58:43.373 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:58:43.376 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.375804 25215 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46598:
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c1_i"
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c2_s"
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c3_double"
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DOUBLE
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c4_long"
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c5_bool"
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c6_short"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT16
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c7_float"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: FLOAT
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c8_binary"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BINARY
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c9_unixtime_micros"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: UNIXTIME_MICROS
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.380 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x59b355db, L:/127.0.0.1:53382 - R:/127.24.131.124:42491] using SASL/PLAIN
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c10_byte"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c11_decimal32"
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL32
06:58:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 9
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c12_decimal64"
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL64
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 18
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c13_decimal128"
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL128
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 38
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c14_varchar"
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: VARCHAR
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) length: 65535
06:58:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c15_date"
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DATE
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
06:58:43.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 7000517cc2ca45fba783de2b5263b170
06:58:43.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.392855 25427 tablet_service.cc:1467] Processing CreateTablet for tablet 9d42494c2ae64abab1916910a688a388 (DEFAULT_TABLE table=test [id=7000517cc2ca45fba783de2b5263b170]), partition=RANGE (key) PARTITION 50 <= VALUES
06:58:43.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.393200 25427 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d42494c2ae64abab1916910a688a388. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.395468 25831 tablet_bootstrap.cc:492] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: Bootstrap starting.
06:58:43.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.395856 25580 tablet_service.cc:1467] Processing CreateTablet for tablet f5320d9d40fe4aedbdb8229f9c6d764d (DEFAULT_TABLE table=test [id=7000517cc2ca45fba783de2b5263b170]), partition=RANGE (key) PARTITION VALUES < 50
06:58:43.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.396188 25580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5320d9d40fe4aedbdb8229f9c6d764d. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:43.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.396301 25831 tablet_bootstrap.cc:654] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: Neither blocks nor log segments found. Creating new log.
06:58:43.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.396855 25831 log.cc:824] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: Log is configured to *not* fsync() on all Append() calls
06:58:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.397673 25831 tablet_bootstrap.cc:492] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: No bootstrap required, opened a new log
06:58:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.397753 25831 ts_tablet_manager.cc:1367] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.399778 25831 raft_consensus.cc:348] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" member_type: VOTER last_known_addr { host: "127.24.131.65" port: 37653 } }
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.399883 25833 tablet_bootstrap.cc:492] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: Bootstrap starting.
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.399930 25831 raft_consensus.cc:374] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.399963 25831 raft_consensus.cc:729] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe51ffe4ac064164bf25f1c5653c94d0, State: Initialized, Role: FOLLOWER
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400082 25831 consensus_queue.cc:260] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" member_type: VOTER last_known_addr { host: "127.24.131.65" port: 37653 } }
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400158 25831 raft_consensus.cc:388] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400192 25831 raft_consensus.cc:482] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400238 25831 raft_consensus.cc:3037] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 0 FOLLOWER]: Advancing to term 1
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400578 25833 tablet_bootstrap.cc:654] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: Neither blocks nor log segments found. Creating new log.
06:58:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400841 25831 raft_consensus.cc:504] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" member_type: VOTER last_known_addr { host: "127.24.131.65" port: 37653 } }
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.400943 25831 leader_election.cc:302] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: fe51ffe4ac064164bf25f1c5653c94d0; no voters:
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401080 25833 log.cc:824] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: Log is configured to *not* fsync() on all Append() calls
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401105 25831 leader_election.cc:288] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [CANDIDATE]: Term 1 election: Requested vote from peers
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401296 25831 ts_tablet_manager.cc:1398] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401456 25513 heartbeater.cc:503] Master 127.24.131.125:32773 was elected leader, sending a full tablet report...
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401463 25834 raft_consensus.cc:2781] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 1 FOLLOWER]: Leader election won for term 1
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401629 25834 raft_consensus.cc:686] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [term 1 LEADER]: Becoming Leader. State: Replica: fe51ffe4ac064164bf25f1c5653c94d0, State: Running, Role: LEADER
06:58:43.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.401717 25834 consensus_queue.cc:237] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" member_type: VOTER last_known_addr { host: "127.24.131.65" port: 37653 } }
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.402380 25833 tablet_bootstrap.cc:492] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: No bootstrap required, opened a new log
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.402451 25833 ts_tablet_manager.cc:1367] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.402753 25215 catalog_manager.cc:5484] T 9d42494c2ae64abab1916910a688a388 P fe51ffe4ac064164bf25f1c5653c94d0 reported cstate change: term changed from 0 to 1, leader changed from <none> to fe51ffe4ac064164bf25f1c5653c94d0 (127.24.131.65). New cstate: current_term: 1 leader_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "fe51ffe4ac064164bf25f1c5653c94d0" member_type: VOTER last_known_addr { host: "127.24.131.65" port: 37653 } health_report { overall_health: HEALTHY } } }
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.404521 25833 raft_consensus.cc:348] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } }
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.404659 25833 raft_consensus.cc:374] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.404687 25833 raft_consensus.cc:729] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 436da135bace402ba6dede0b5d9f6ab5, State: Initialized, Role: FOLLOWER
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.404866 25833 consensus_queue.cc:260] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } }
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.404968 25839 raft_consensus.cc:482] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405048 25839 raft_consensus.cc:504] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } }
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405153 25839 leader_election.cc:302] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 436da135bace402ba6dede0b5d9f6ab5; no voters:
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405156 25833 raft_consensus.cc:388] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405313 25833 raft_consensus.cc:482] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405340 25833 raft_consensus.cc:3037] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 0 FOLLOWER]: Advancing to term 1
06:58:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.405354 25839 leader_election.cc:288] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers
06:58:43.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406118 25833 raft_consensus.cc:504] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } }
06:58:43.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406350 25833 leader_election.cc:302] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 436da135bace402ba6dede0b5d9f6ab5; no voters:
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406391 25839 raft_consensus.cc:2741] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406414 25833 leader_election.cc:288] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [CANDIDATE]: Term 1 election: Requested vote from peers
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406543 25840 raft_consensus.cc:2781] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 1 FOLLOWER]: Leader election won for term 1
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406635 25833 ts_tablet_manager.cc:1398] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406664 25840 raft_consensus.cc:686] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [term 1 LEADER]: Becoming Leader. State: Replica: 436da135bace402ba6dede0b5d9f6ab5, State: Running, Role: LEADER
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406738 25840 consensus_queue.cc:237] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } }
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.406796 25648 heartbeater.cc:503] Master 127.24.131.125:32773 was elected leader, sending a full tablet report...
06:58:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.407521 25215 catalog_manager.cc:5484] T f5320d9d40fe4aedbdb8229f9c6d764d P 436da135bace402ba6dede0b5d9f6ab5 reported cstate change: term changed from 0 to 1, leader changed from <none> to 436da135bace402ba6dede0b5d9f6ab5 (127.24.131.66). New cstate: current_term: 1 leader_uuid: "436da135bace402ba6dede0b5d9f6ab5" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "436da135bace402ba6dede0b5d9f6ab5" member_type: VOTER last_known_addr { host: "127.24.131.66" port: 35223 } health_report { overall_health: HEALTHY } } }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.420821 25215 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46598:
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "val"
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:58:43.425 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 9cfb1d61cdc342b0a8a961c124af48b4
06:58:43.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.428840 25715 tablet_service.cc:1467] Processing CreateTablet for tablet a1d0a573f49a42b5b4077a9ecdf625b7 (DEFAULT_TABLE table=simple-test [id=9cfb1d61cdc342b0a8a961c124af48b4]), partition=RANGE (key) PARTITION UNBOUNDED
06:58:43.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.429167 25715 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1d0a573f49a42b5b4077a9ecdf625b7. 1 dirs total, 0 dirs full, 0 dirs failed
06:58:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.431617 25847 tablet_bootstrap.cc:492] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: Bootstrap starting.
06:58:43.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.432286 25847 tablet_bootstrap.cc:654] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: Neither blocks nor log segments found. Creating new log.
06:58:43.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.432533 25847 log.cc:824] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: Log is configured to *not* fsync() on all Append() calls
06:58:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.433153 25847 tablet_bootstrap.cc:492] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: No bootstrap required, opened a new log
06:58:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.433218 25847 ts_tablet_manager.cc:1367] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
06:58:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.434707 25847 raft_consensus.cc:348] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f0d6561d1e1145b78f710614a3187cb4" member_type: VOTER last_known_addr { host: "127.24.131.67" port: 36017 } }
06:58:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.434827 25847 raft_consensus.cc:374] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.434844 25847 raft_consensus.cc:729] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f0d6561d1e1145b78f710614a3187cb4, State: Initialized, Role: FOLLOWER
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.434943 25847 consensus_queue.cc:260] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f0d6561d1e1145b78f710614a3187cb4" member_type: VOTER last_known_addr { host: "127.24.131.67" port: 36017 } }
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435010 25847 raft_consensus.cc:388] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435045 25847 raft_consensus.cc:482] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435086 25847 raft_consensus.cc:3037] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 0 FOLLOWER]: Advancing to term 1
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435596 25847 raft_consensus.cc:504] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f0d6561d1e1145b78f710614a3187cb4" member_type: VOTER last_known_addr { host: "127.24.131.67" port: 36017 } }
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435684 25847 leader_election.cc:302] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f0d6561d1e1145b78f710614a3187cb4; no voters:
06:58:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435866 25847 leader_election.cc:288] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [CANDIDATE]: Term 1 election: Requested vote from peers
06:58:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.435925 25849 raft_consensus.cc:2781] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 1 FOLLOWER]: Leader election won for term 1
06:58:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.436051 25847 ts_tablet_manager.cc:1398] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
06:58:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.436288 25783 heartbeater.cc:503] Master 127.24.131.125:32773 was elected leader, sending a full tablet report...
06:58:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.436103 25849 raft_consensus.cc:686] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [term 1 LEADER]: Becoming Leader. State: Replica: f0d6561d1e1145b78f710614a3187cb4, State: Running, Role: LEADER
06:58:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.436437 25849 consensus_queue.cc:237] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f0d6561d1e1145b78f710614a3187cb4" member_type: VOTER last_known_addr { host: "127.24.131.67" port: 36017 } }
06:58:43.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.437248 25215 catalog_manager.cc:5484] T a1d0a573f49a42b5b4077a9ecdf625b7 P f0d6561d1e1145b78f710614a3187cb4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f0d6561d1e1145b78f710614a3187cb4 (127.24.131.67). New cstate: current_term: 1 leader_uuid: "f0d6561d1e1145b78f710614a3187cb4" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f0d6561d1e1145b78f710614a3187cb4" member_type: VOTER last_known_addr { host: "127.24.131.67" port: 36017 } health_report { overall_health: HEALTHY } } }
06:58:43.442 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f5320d9d40fe4aedbdb8229f9c6d764d for table 'test' with partition [<start>, 0x80000032)
06:58:43.443 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:43.443 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x80000032, ttl=299999, tablet=f5320d9d40fe4aedbdb8229f9c6d764d@[436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)[L]]}, Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:43.443 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=null) after lookup
06:58:43.445 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)] connecting to peer
06:58:43.446 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)] Successfully connected to peer
06:58:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.452033 25514 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:9d42494c2ae64abab1916910a688a388)
06:58:43.458 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:58:43.460 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfd3f0246, L:/127.0.0.1:57224 - R:/127.24.131.66:35223] using TOKEN/null
06:58:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.477851 25784 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:a1d0a573f49a42b5b4077a9ecdf625b7)
06:58:43.486 [INFO - main] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-task31bC9m/java/build/dist-test/spark-warehouse').
06:58:43.486 [INFO - main] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-task31bC9m/java/build/dist-test/spark-warehouse'.
06:58:43.495 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 7000517cc2ca45fba783de2b5263b170
06:58:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:58:43.597518 25649 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:f5320d9d40fe4aedbdb8229f9c6d764d)
06:58:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.633064 25342 catalog_manager.cc:1259] Loaded cluster ID: 421afb2b5e1d4349a7c4cd77eafb3c90
06:58:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.633165 25342 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: loading cluster ID for follower catalog manager: success
06:58:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.634552 25342 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: acquiring CA information for follower catalog manager: success
06:58:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.634974 25342 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2a44e998012d4bdd9a1ca8db4ae66364: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:58:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.692938 25356 catalog_manager.cc:1259] Loaded cluster ID: 421afb2b5e1d4349a7c4cd77eafb3c90
06:58:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.693028 25356 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: loading cluster ID for follower catalog manager: success
06:58:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.694460 25356 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: acquiring CA information for follower catalog manager: success
06:58:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.694839 25356 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ad569656bca0471bb66afda61519c288: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:58:43.798 [DEBUG - main] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:43.800 [DEBUG - kudu-nio-0] (Connection.java:757) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] Successfully connected to peer
06:58:43.811 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
06:58:43.813 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x67c8908a, L:/127.0.0.1:45064 - R:/127.24.131.65:37653] using TOKEN/null
06:58:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.847982 25649 maintenance_manager.cc:419] P 436da135bace402ba6dede0b5d9f6ab5: Scheduling FlushMRSOp(f5320d9d40fe4aedbdb8229f9c6d764d): perf score=1.000000
06:58:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.883172 25537 maintenance_manager.cc:643] P 436da135bace402ba6dede0b5d9f6ab5: FlushMRSOp(f5320d9d40fe4aedbdb8229f9c6d764d) complete. Timing: real 0.035s user 0.009s sys 0.000s Metrics: {"bytes_written":39822,"cfile_init":1,"compiler_manager_pool.queue_time_us":1361,"drs_written":1,"lbm_read_time_us":31,"lbm_reads_lt_1ms":4,"lbm_write_time_us":624,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":50,"thread_start_us":151,"threads_started":2}
06:58:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.883445 25649 maintenance_manager.cc:419] P 436da135bace402ba6dede0b5d9f6ab5: Scheduling UndoDeltaBlockGCOp(f5320d9d40fe4aedbdb8229f9c6d764d): 1548 bytes on disk
06:58:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.883641 25537 maintenance_manager.cc:643] P 436da135bace402ba6dede0b5d9f6ab5: UndoDeltaBlockGCOp(f5320d9d40fe4aedbdb8229f9c6d764d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:58:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.952823 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.000000
06:58:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.981629 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.029s user 0.009s sys 0.000s Metrics: {"bytes_written":39534,"cfile_init":1,"compiler_manager_pool.queue_time_us":126,"drs_written":1,"lbm_read_time_us":25,"lbm_reads_lt_1ms":4,"lbm_write_time_us":634,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28,"thread_start_us":159,"threads_started":2}
06:58:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.982066 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 446 bytes on disk
06:58:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:43.982411 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:58:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.232769 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.000000
06:58:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.237094 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.004s sys 0.000s Metrics: {"bytes_written":39841,"cfile_init":1,"dirs.queue_time_us":441,"drs_written":1,"lbm_read_time_us":18,"lbm_reads_lt_1ms":4,"lbm_write_time_us":560,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":44,"spinlock_wait_cycles":4480}
06:58:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.237442 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 625 bytes on disk
06:58:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.237797 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:58:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.488329 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.953555
06:58:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.492494 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.003s sys 0.000s Metrics: {"bytes_written":40201,"cfile_init":1,"dirs.queue_time_us":59,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":560,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":72}
06:58:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.492767 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 927 bytes on disk
06:58:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.493081 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":25,"lbm_reads_lt_1ms":4}
06:58:44.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.743379 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.953555
06:58:44.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.747607 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.000s sys 0.003s Metrics: {"bytes_written":40563,"cfile_init":1,"dirs.queue_time_us":650,"drs_written":1,"lbm_read_time_us":25,"lbm_reads_lt_1ms":4,"lbm_write_time_us":655,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":89,"spinlock_wait_cycles":768}
06:58:44.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.747815 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1106 bytes on disk
06:58:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.747949 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:58:44.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:44.998318 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.953466
06:58:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.002408 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.001s sys 0.002s Metrics: {"bytes_written":40521,"cfile_init":1,"dirs.queue_time_us":42,"drs_written":1,"lbm_read_time_us":18,"lbm_reads_lt_1ms":4,"lbm_write_time_us":598,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":87}
06:58:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.002566 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1082 bytes on disk
06:58:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.002710 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:58:45.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.252959 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=2.945743
06:58:45.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.257845 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.005s user 0.004s sys 0.000s Metrics: {"bytes_written":40616,"cfile_init":1,"dirs.queue_time_us":42,"dirs.run_cpu_time_us":214,"dirs.run_wall_time_us":1539,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":660,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":95}
06:58:45.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.258066 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1164 bytes on disk
06:58:45.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.258286 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":23,"lbm_reads_lt_1ms":4}
06:58:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.508652 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=2.945654
06:58:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.513374 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.005s user 0.003s sys 0.000s Metrics: {"bytes_written":40833,"cfile_init":1,"dirs.queue_time_us":62,"dirs.run_cpu_time_us":200,"dirs.run_wall_time_us":1587,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":567,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":97}
06:58:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.513576 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1189 bytes on disk
06:58:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.513713 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":24,"lbm_reads_lt_1ms":4}
06:58:45.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.763999 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.953555
06:58:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.768391 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.003s sys 0.001s Metrics: {"bytes_written":40634,"cfile_init":1,"dirs.queue_time_us":44,"dirs.run_cpu_time_us":215,"dirs.run_wall_time_us":1583,"drs_written":1,"lbm_read_time_us":19,"lbm_reads_lt_1ms":4,"lbm_write_time_us":619,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":89}
06:58:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.768746 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1107 bytes on disk
06:58:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:45.768993 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":22,"lbm_reads_lt_1ms":4}
06:58:46.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.019264 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=2.945654
06:58:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.023197 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.000s sys 0.003s Metrics: {"bytes_written":40590,"cfile_init":1,"dirs.queue_time_us":39,"drs_written":1,"lbm_read_time_us":14,"lbm_reads_lt_1ms":4,"lbm_write_time_us":597,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":104}
06:58:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.023602 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1258 bytes on disk
06:58:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.023936 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":16,"lbm_reads_lt_1ms":4}
06:58:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.274250 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=2.945654
06:58:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.278340 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.003s sys 0.000s Metrics: {"bytes_written":40766,"cfile_init":1,"dirs.queue_time_us":49,"drs_written":1,"lbm_read_time_us":13,"lbm_reads_lt_1ms":4,"lbm_write_time_us":522,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":99}
06:58:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.278789 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1206 bytes on disk
06:58:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.279091 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":21,"lbm_reads_lt_1ms":4}
06:58:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.529395 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=2.945743
06:58:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.533385 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.004s user 0.003s sys 0.000s Metrics: {"bytes_written":40783,"cfile_init":1,"dirs.queue_time_us":32,"drs_written":1,"lbm_read_time_us":36,"lbm_reads_lt_1ms":4,"lbm_write_time_us":511,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":103}
06:58:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.533586 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 1249 bytes on disk
06:58:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.533771 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":22,"lbm_reads_lt_1ms":4}
06:58:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.784205 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling FlushMRSOp(9d42494c2ae64abab1916910a688a388): perf score=1.000000
06:58:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.787307 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: FlushMRSOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.003s user 0.002s sys 0.000s Metrics: {"bytes_written":39899,"cfile_init":1,"dirs.queue_time_us":85,"drs_written":1,"lbm_read_time_us":13,"lbm_reads_lt_1ms":4,"lbm_write_time_us":477,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43,"spinlock_wait_cycles":896}
06:58:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.787513 25514 maintenance_manager.cc:419] P fe51ffe4ac064164bf25f1c5653c94d0: Scheduling UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388): 599 bytes on disk
06:58:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:58:46.787726 25402 maintenance_manager.cc:643] P fe51ffe4ac064164bf25f1c5653c94d0: UndoDeltaBlockGCOp(9d42494c2ae64abab1916910a688a388) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":26,"lbm_reads_lt_1ms":4}
06:58:51.656 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 7000517cc2ca45fba783de2b5263b170
06:58:51.745 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for test
06:58:51.745 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2206) Key ranges for test
06:58:51.746 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, <end>), 35642, "f5320d9d40fe4aedbdb8229f9c6d764d" [<start>, 0x80000032)
06:58:51.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [<start>, 0x8000004D), 35354, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000004D, 0x80000079), 35661, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000079, 0x800000C1), 36021, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800000C1, 0x8000011A), 36383, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000011A, 0x80000171), 36341, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000171, 0x800001D0), 36436, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800001D0, 0x80000231), 36653, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000231, 0x8000028A), 36454, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x8000028A, 0x800002F2), 36410, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800002F2, 0x80000355), 36586, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x80000355, 0x800003BC), 36603, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2213) [0x800003BC, <end>), 35719, "9d42494c2ae64abab1916910a688a388" [0x80000032, <end>)
06:58:51.762 [INFO - main] (Logging.scala:57) Starting job: collectAsList at SparkSQLTest.scala:471
06:58:51.763 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (collectAsList at SparkSQLTest.scala:471) with 13 output partitions
06:58:51.763 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (collectAsList at SparkSQLTest.scala:471)
06:58:51.763 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
06:58:51.763 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
06:58:51.765 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471), which has no missing parents
06:58:51.771 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 25.2 KiB, free 1993.9 MiB)
06:58:51.774 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 9.6 KiB, free 1993.8 MiB)
06:58:51.775 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-s24k.c.gcp-upstream.internal:41561 (size: 9.6 KiB, free: 1993.9 MiB)
06:58:51.776 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
06:58:51.776 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 13 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at collectAsList at SparkSQLTest.scala:471) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))
06:58:51.776 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 13 tasks resource profile 0
06:58:51.779 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-s24k.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5316 bytes) taskResourceAssignments Map()
06:58:51.781 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
06:58:51.788 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:58:51.818 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 50 rows, more = false, responseScanTimestamp = 7021013941416747008) for scanner KuduScanner(table=test, tablet=f5320d9d40fe4aedbdb8229f9c6d764d, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:58:51.861 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 17959 bytes result sent to driver
06:58:51.863 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-s24k.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5322 bytes) taskResourceAssignments Map()
06:58:51.864 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 86 ms on dist-test-slave-s24k.c.gcp-upstream.internal (executor driver) (1/13)
06:58:51.864 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
06:58:51.869 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:58:51.900 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false, responseScanTimestamp = 7021013941803024384) for scanner KuduScanner(table=test, tablet=9d42494c2ae64abab1916910a688a388, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x8000004D, lastPrimaryKey=<last>)
06:58:51.923 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1). 12267 bytes result sent to driver
06:58:51.925 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-s24k.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5328 bytes) taskResourceAssignments Map()
06:58:51.925 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
06:58:51.925 [INFO - task-result-getter-1] (Logging.scala:57) Finished task 1.0 in stage 0.0 (TID 1) in 62 ms on dist-test-slave-s24k.c.gcp-upstream.internal (executor driver) (2/13)
06:58:51.931 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:58:51.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714114731 (unix time) try "date -d @1714114731" if you are using GNU date ***
06:58:51.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @ 0x0 (unknown)
06:58:51.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 25383 (TID 0x7fd363139700) from PID 8; stack trace: ***
06:58:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3788af980 (unknown)
06:58:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd37874b68e kudu::ScanSpec::SetLowerBoundKey()
06:58:51.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3791e42c3 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
06:58:51.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3791eb614 kudu::tablet::CFileSet::Iterator::Init()
06:58:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd37927aa9b kudu::tablet::DeltaApplier::Init()
06:58:51.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd378700ef2 kudu::MaterializingIterator::Init()
06:58:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3786f7c80 kudu::InitAndMaybeWrap()
06:58:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3786f9995 kudu::UnionIterator::InitSubIterators()
06:58:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3786fc3fa kudu::UnionIterator::Init()
06:58:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3791865c7 kudu::tablet::Tablet::Iterator::Init()
06:58:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd379562b0d kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
06:58:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd379564b0f kudu::tserver::TabletServiceImpl::Scan()
06:58:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd378b4c6f8 kudu::rpc::GeneratedServiceIf::Handle()
06:58:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd378b4d6cc kudu::rpc::ServicePool::RunThread()
06:58:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd378582394 kudu::Thread::SuperviseThread()
06:58:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3788a46db start_thread
06:58:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fd3769e871f clone
06:58:52.159 [DEBUG - kudu-nio-0] (Connection.java:239) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] handling channelInactive
06:58:52.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state READY due to: connection closed
06:58:52.159 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: connection closed
06:58:52.180 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:52.180 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:52.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=2, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=2, TimeoutTracker(timeout=30000, elapsed=249), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2)))) after lookup
06:58:52.182 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:52.183 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.183 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.183 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.201 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:52.201 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:52.201 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=4, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=4, TimeoutTracker(timeout=30000, elapsed=269), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612)))) after lookup
06:58:52.202 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:52.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.204 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.204 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.240 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:52.240 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:52.240 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=6, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=6, TimeoutTracker(timeout=30000, elapsed=308), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727)))) after lookup
06:58:52.241 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:52.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.242 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:52.340 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:52.341 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=8, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=8, TimeoutTracker(timeout=30000, elapsed=409), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291)))) after lookup
06:58:52.342 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:52.344 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.345 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.520 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:52.520 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:52.521 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=10, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=10, TimeoutTracker(timeout=30000, elapsed=588), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372)))) after lookup
06:58:52.522 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:52.523 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.523 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:52.523 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.440 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:53.441 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:53.441 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=12, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=12, TimeoutTracker(timeout=30000, elapsed=1509), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757)))) after lookup
06:58:53.442 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:53.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.444 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:53.621 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:53.621 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=14, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=14, TimeoutTracker(timeout=30000, elapsed=1689), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271)))) after lookup
06:58:53.622 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:53.623 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.623 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.623 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.800 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:53.801 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:53.801 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=16, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=16, TimeoutTracker(timeout=30000, elapsed=1869), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512)))) after lookup
06:58:53.802 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:53.804 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.804 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:53.804 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:54.780 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:54.780 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:54.781 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=18, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=18, TimeoutTracker(timeout=30000, elapsed=2849), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030)))) after lookup
06:58:54.783 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:54.784 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:54.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:54.785 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:55.340 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:55.340 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:55.341 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=20, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=20, TimeoutTracker(timeout=30000, elapsed=3409), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405)))) after lookup
06:58:55.343 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:55.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:55.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:55.345 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:57.119 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:57.120 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:57.120 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=22, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=22, TimeoutTracker(timeout=30000, elapsed=5188), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183)))) after lookup
06:58:57.122 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:57.123 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:57.123 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:57.123 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:58.601 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:58:58.601 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:58:58.602 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=24, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=24, TimeoutTracker(timeout=30000, elapsed=6670), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666)))) after lookup
06:58:58.604 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:58:58.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:58.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:58:58.606 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:01.720 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:01.721 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:01.721 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=26, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=26, TimeoutTracker(timeout=30000, elapsed=9789), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509)))) after lookup
06:59:01.723 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:01.724 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:01.724 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:01.724 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.680 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:02.681 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:02.681 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=28, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=28, TimeoutTracker(timeout=30000, elapsed=10749), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122)))) after lookup
06:59:02.683 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:02.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.685 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:02.880 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:02.881 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=30, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=30, TimeoutTracker(timeout=30000, elapsed=10949), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937)))) after lookup
06:59:02.882 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:02.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:02.884 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:06.299 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:06.300 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:06.300 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=32, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=32, TimeoutTracker(timeout=30000, elapsed=14369), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309)))) after lookup
06:59:06.303 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:06.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:06.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:06.305 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:10.260 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:10.260 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:10.261 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=34, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=34, TimeoutTracker(timeout=30000, elapsed=18329), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117)))) after lookup
06:59:10.264 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:10.265 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:10.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:10.265 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:12.419 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:12.420 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:12.420 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=36, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=36, TimeoutTracker(timeout=30000, elapsed=20488), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305)))) after lookup
06:59:12.421 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:12.423 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:12.423 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:12.423 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:13.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:13.660 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:13.660 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=38, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=38, TimeoutTracker(timeout=30000, elapsed=21728), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541)))) after lookup
06:59:13.662 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:13.663 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:13.663 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:13.663 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.319 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:17.320 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:17.320 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=40, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=40, TimeoutTracker(timeout=30000, elapsed=25388), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681)))) after lookup
06:59:17.322 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:17.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.324 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:17.940 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:17.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=42, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=42, TimeoutTracker(timeout=30000, elapsed=26009), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508)))) after lookup
06:59:17.942 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:17.943 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.943 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:17.943 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:18.939 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:18.940 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:18.940 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=44, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=44, TimeoutTracker(timeout=30000, elapsed=27008), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343)))) after lookup
06:59:18.942 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:18.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:18.944 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:18.944 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.019 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:19.020 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:19.020 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=46, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=46, TimeoutTracker(timeout=30000, elapsed=27089), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046)))) after lookup
06:59:19.023 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:19.024 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.025 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.026 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
06:59:19.027 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
06:59:19.045 [ERROR - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:94) Exception in task 2.0 in stage 0.0 (TID 2)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160) ~[main/:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
06:59:19.139 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 3.0 in stage 0.0 (TID 3) (dist-test-slave-s24k.c.gcp-upstream.internal, executor driver, partition 3, ANY, 5328 bytes) taskResourceAssignments Map()
06:59:19.139 [INFO - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (Logging.scala:57) Running task 3.0 in stage 0.0 (TID 3)
06:59:19.142 [WARN - task-result-getter-2] (Logging.scala:69) Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-s24k.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
06:59:19.145 [DEBUG - Executor task launch worker for task 3.0 in stage 0.0 (TID 3)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
06:59:19.148 [ERROR - task-result-getter-2] (Logging.scala:73) Task 2 in stage 0.0 failed 1 times; aborting job
06:59:19.149 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:19.149 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:19.150 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=null)) after lookup
06:59:19.150 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:19.153 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.153 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.154 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.154 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
06:59:19.155 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
06:59:19.159 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:19.160 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:19.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=3, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=3, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [4ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [7ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [8ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [13ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@1080880207(state=PENDING, result=null, callback=(continuation of Deferred@849673989 after retry RPC@1761134767), errback=(continuation of Deferred@849673989 after retry RPC@1761134767)))) after lookup
06:59:19.160 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:19.161 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
06:59:19.162 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Executor is trying to kill task 3.0 in stage 0.0 (TID 3), reason: Stage cancelled
06:59:19.162 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.162 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (collectAsList at SparkSQLTest.scala:471) failed in 27.394 s due to Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-s24k.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
06:59:19.162 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.164 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.165 [INFO - main] (Logging.scala:57) Job 0 failed: collectAsList at SparkSQLTest.scala:471, took 27.402600 s
06:59:19.177 [INFO - dispatcher-event-loop-1] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
06:59:19.180 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9d42494c2ae64abab1916910a688a388 for table 'test' with partition [0x80000032, <end>)
06:59:19.180 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x80000032, upperBoundPartitionKey=0x, ttl=299999, tablet=9d42494c2ae64abab1916910a688a388@[fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)[L]]}]
06:59:19.181 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=5, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=5, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [4ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [7ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [8ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [13ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [18ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [34ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, deferred=Deferred@1080880207(state=PENDING, result=null, callback=(continuation of Deferred@849673989 after retry RPC@1761134767) -> (continuation of Deferred@1729928213 after retry RPC@87688433), errback=(continuation of Deferred@849673989 after retry RPC@1761134767) -> (continuation of Deferred@1729928213 after retry RPC@87688433)))) after lookup
06:59:19.182 [INFO - main] (Logging.scala:57) MemoryStore cleared
06:59:19.182 [INFO - main] (Logging.scala:57) BlockManager stopped
06:59:19.182 [DEBUG - kudu-nio-1] (Connection.java:748) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] connecting to peer
06:59:19.184 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
06:59:19.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] cleaning up while in state CONNECTING due to: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.184 [INFO - dispatcher-event-loop-0] (Logging.scala:57) OutputCommitCoordinator stopped!
06:59:19.184 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653) for tablet 9d42494c2ae64abab1916910a688a388: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
06:59:19.189 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
06:59:19.189 [DEBUG - main] (Connection.java:575) [peer 436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)] disconnecting while in state READY
06:59:19.189 [DEBUG - main] (Connection.java:575) [peer master-127.24.131.126:46597(127.24.131.126:46597)] disconnecting while in state READY
06:59:19.189 [DEBUG - main] (Connection.java:575) [peer master-127.24.131.125:32773(127.24.131.125:32773)] disconnecting while in state READY
06:59:19.190 [DEBUG - main] (Connection.java:575) [peer master-127.24.131.124:42491(127.24.131.124:42491)] disconnecting while in state READY
06:59:19.190 [DEBUG - main] (Connection.java:575) [peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653)] disconnecting while in state TERMINATED
06:59:19.191 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)] handling channelInactive
06:59:19.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 436da135bace402ba6dede0b5d9f6ab5(127.24.131.66:35223)] cleaning up while in state READY due to: connection closed
06:59:19.191 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.131.125:32773(127.24.131.125:32773)] handling channelInactive
06:59:19.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.131.125:32773(127.24.131.125:32773)] cleaning up while in state READY due to: connection closed
06:59:19.191 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.131.126:46597(127.24.131.126:46597)] handling channelInactive
06:59:19.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.131.126:46597(127.24.131.126:46597)] cleaning up while in state READY due to: connection closed
06:59:19.192 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:19.192 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.131.124:42491(127.24.131.124:42491)] handling channelInactive
06:59:19.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.131.124:42491(127.24.131.124:42491)] cleaning up while in state READY due to: connection closed
06:59:19.193 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 40 thread-local buffer(s) from thread: kudu-nio-1
06:59:19.193 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 30 thread-local buffer(s) from thread: kudu-nio-0
06:59:19.206 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.207616 25101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task31bC9m/build/release/bin/kudu with pid 25518
06:59:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.215898 25101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task31bC9m/build/release/bin/kudu with pid 25653
06:59:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.221145 25101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task31bC9m/build/release/bin/kudu with pid 25112
06:59:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.226635 25101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task31bC9m/build/release/bin/kudu with pid 25181
06:59:19.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.232074 25101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task31bC9m/build/release/bin/kudu with pid 25256
06:59:19.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:19Z chronyd exiting
06:59:19.286 [ERROR - main] (RetryRule.java:206) org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange: failed attempt 1
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-s24k.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDD.collect(RDD.scala:1029) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471) ~[test/:?]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[test/:?]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[test/:?]
at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (dist-test-slave-s24k.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.spark.kudu.RowIterator.hasNext(KuduRDD.scala:160)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, KuduRpc(method=Scan, tablet=9d42494c2ae64abab1916910a688a388, attempt=47, TimeoutTracker(timeout=30000, elapsed=27092), Traces: [0ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [227ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: connection closed, [227ms] delaying RPC due to: Network error: connection closed, [247ms] refreshing cache from master, [247ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [248ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [249ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [251ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [251ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [267ms] refreshing cache from master, [267ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [268ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [270ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [272ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [273ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [306ms] refreshing cache from master, [306ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [307ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [309ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [310ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [310ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [407ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [413ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [586ms] refreshing cache from master, [586ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [588ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [590ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [591ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [591ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1507ms] refreshing cache from master, [1507ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1508ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1510ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1512ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1512ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1687ms] refreshing cache from master, [1687ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1688ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1690ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1691ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1691ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1867ms] refreshing cache from master, [1867ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [1868ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [1870ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [1872ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [1872ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2846ms] refreshing cache from master, [2846ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [2848ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [2851ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [2853ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [2853ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3406ms] refreshing cache from master, [3407ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [3408ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [3410ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [3413ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [3414ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5186ms] refreshing cache from master, [5186ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [5187ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [5190ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [5191ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [5191ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6667ms] refreshing cache from master, [6668ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [6669ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [6672ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [6674ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [6674ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9787ms] refreshing cache from master, [9787ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [9788ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [9790ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [9792ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [9792ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10747ms] refreshing cache from master, [10747ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10748ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10751ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10753ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10753ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10947ms] refreshing cache from master, [10947ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [10948ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [10950ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [10952ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [10952ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14366ms] refreshing cache from master, [14366ms] Sub RPC GetTableLocations: sending RPC to server master-127.24.131.125:32773, [14367ms] Sub RPC GetTableLocations: received response from server master-127.24.131.125:32773: OK, [14371ms] sending RPC to server fe51ffe4ac064164bf25f1c5653c94d0, [14373ms] received response from server fe51ffe4ac064164bf25f1c5653c94d0: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [14373ms] delaying RPC due to: Network error: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653, [18326ms] refreshing cache from master, [18326ms] too many traces: truncated at 100 traces, deferred=Deferred@329085688(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2) -> (continuation of Deferred@2085046358 after retry RPC@528859612) -> (continuation of Deferred@875450295 after retry RPC@93643727) -> (continuation of Deferred@572122 after retry RPC@1990266291) -> (continuation of Deferred@1370616124 after retry RPC@788771372) -> (continuation of Deferred@418406832 after retry RPC@1168596757) -> (continuation of Deferred@1849920285 after retry RPC@300105271) -> (continuation of Deferred@219410547 after retry RPC@1586669512) -> (continuation of Deferred@275066237 after retry RPC@1535272030) -> (continuation of Deferred@1783172855 after retry RPC@672656405) -> (continuation of Deferred@58865054 after retry RPC@350357183) -> (continuation of Deferred@1529148707 after retry RPC@1602502666) -> (continuation of Deferred@1045837101 after retry RPC@1127200509) -> (continuation of Deferred@246891358 after retry RPC@1479649122) -> (continuation of Deferred@1689790411 after retry RPC@837260937) -> (continuation of Deferred@262787424 after retry RPC@1088400309) -> (continuation of Deferred@2020427646 after retry RPC@1171442117) -> (continuation of Deferred@1988318197 after retry RPC@2082787305) -> (continuation of Deferred@944969353 after retry RPC@1657986541) -> (continuation of Deferred@153181474 after retry RPC@1285299681) -> (continuation of Deferred@1998013322 after retry RPC@1531460508) -> (continuation of Deferred@1008156434 after retry RPC@1088173343) -> (continuation of Deferred@764359421 after retry RPC@215654046) -> (continuation of Deferred@1128759808 after retry RPC@24916026))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 19 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer fe51ffe4ac064164bf25f1c5653c94d0(127.24.131.65:37653): Connection refused: /127.24.131.65:37653
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.131.65:37653
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
at scala.Option.foreach(Option.scala:407)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)
at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)
at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3696)
at org.apache.spark.sql.Dataset.$anonfun$collectAsList$1(Dataset.scala:2977)
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)
at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2976)
at org.apache.kudu.spark.kudu.SparkSQLTest.$anonfun$testScanWithKeyRange$1(SparkSQLTest.scala:471)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
at org.apache.kudu.spark.kudu.SparkSQLTest.testScanWithKeyRange(SparkSQLTest.scala:468)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)