07:02:30.680 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:02:30.680 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBMRGjA/test-tmp
07:02:30.680 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
07:02:30.680 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBMRGjA/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
07:02:30.682 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257"
extra_tserver_flags: "--flush_threshold_mb=1"
extra_tserver_flags: "--flush_threshold_secs=1"
extra_tserver_flags: "--enable_rowset_compaction=false"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:02:31.013 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:02:31.013 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:02:31.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.015563 9028 mini_oidc.cc:152] Starting JWKS server
07:02:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.018265 9028 webserver.cc:460] Webserver started at http://127.0.0.1:39113/ using document root <none> and password file <none>
07:02:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.018483 9028 mini_oidc.cc:189] Starting OIDC Discovery server
07:02:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.019013 9028 webserver.cc:460] Webserver started at http://127.0.0.1:46199/ using document root <none> and password file <none>
07:02:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:02:31Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:02:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:02:31Z Disabled control of system clock
07:02:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.053632 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/logs
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data/info.pb
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.62:33719
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.62
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json with env {}
07:02:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.378954 9039 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.379312 9039 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.379429 9039 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387053 9039 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387178 9039 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387224 9039 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387274 9039 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387315 9039 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.387360 9039 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39113/jwks.json
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.396829 9039 master_runner.cc:386] Master server non-default flags:
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.62:33719
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data/info.pb
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.62
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/logs
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:31.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.398218 9039 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.399997 9039 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.410686 9044 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.411304 9045 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.411870 9047 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.412709 9039 server_base.cc:1034] running on GCE node
07:02:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.413547 9039 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.415149 9039 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.416399 9039 hybrid_clock.cc:648] HybridClock initialized: now 1714114951416339 us; error 31 us; skew 500 ppm
07:02:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.416774 9039 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:31.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.419253 9039 webserver.cc:460] Webserver started at http://127.8.209.62:36575/ using document root <none> and password file <none>
07:02:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.420019 9039 fs_manager.cc:359] Metadata directory not provided
07:02:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.420152 9039 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.420492 9039 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.423285 9039 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data/instance:
07:02:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c2b7184b67346f3acfeecc646a6c49f"
07:02:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.424134 9039 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal/instance:
07:02:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c2b7184b67346f3acfeecc646a6c49f"
07:02:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.429443 9039 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.002s
07:02:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.432967 9053 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.434489 9039 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:02:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.434669 9039 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal
07:02:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c2b7184b67346f3acfeecc646a6c49f"
07:02:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.434849 9039 fs_report.cc:383] FS layout report
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data/data
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.454039 9039 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.454893 9039 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.455155 9039 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:31.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.473635 9039 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.62:33719
07:02:31.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.473671 9104 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.62:33719 every 8 connection(s)
07:02:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.475160 9039 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/data/info.pb
07:02:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.479051 9105 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.481346 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9039
07:02:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.481619 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-0/wal/instance
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.484032 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/logs
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data/info.pb
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.61:33187
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.61
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json with env {}
07:02:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.485711 9105 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } has no permanent_uuid. Determining permanent_uuid...
07:02:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.502838 9105 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } has no permanent_uuid. Determining permanent_uuid...
07:02:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.504051 9054 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.505684 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.61:33187: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.555063 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } attempt: 1
07:02:31.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.557191 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.61:33187: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.625440 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } attempt: 2
07:02:31.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.627337 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.61:33187: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.718606 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } attempt: 3
07:02:31.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.720582 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.61:33187: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.812989 9107 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.813320 9107 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.813469 9107 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821107 9107 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821241 9107 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821306 9107 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821395 9107 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821466 9107 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.821537 9107 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39113/jwks.json
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.830770 9107 master_runner.cc:386] Master server non-default flags:
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.61:33187
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data/info.pb
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.61
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/logs
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.832510 9107 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.834280 9107 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.845992 9117 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.845944 9120 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.846894 9118 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.848462 9107 server_base.cc:1034] running on GCE node
07:02:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.849323 9107 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:31.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.850863 9107 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:31.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.852095 9107 hybrid_clock.cc:648] HybridClock initialized: now 1714114951852024 us; error 48 us; skew 500 ppm
07:02:31.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.852455 9107 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.854816 9107 webserver.cc:460] Webserver started at http://127.8.209.61:35031/ using document root <none> and password file <none>
07:02:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.855535 9107 fs_manager.cc:359] Metadata directory not provided
07:02:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.855659 9107 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:31.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.855958 9107 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.858624 9107 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data/instance:
07:02:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5352f3c1b7e643f28d6101dce3817a56"
07:02:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.859418 9107 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal/instance:
07:02:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5352f3c1b7e643f28d6101dce3817a56"
07:02:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.863826 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } attempt: 4
07:02:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.864691 9107 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
07:02:31.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.865756 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.61:33187: Network error: Client connection negotiation failed: client connection to 127.8.209.61:33187: connect: Connection refused (error 111)
07:02:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.868270 9126 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.869577 9107 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
07:02:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.869720 9107 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal
07:02:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5352f3c1b7e643f28d6101dce3817a56"
07:02:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:31 on dist-test-slave-b8wt"
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.869875 9107 fs_report.cc:383] FS layout report
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data/data
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:31.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.905097 9107 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.906009 9107 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.906277 9107 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.925223 9107 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.61:33187
07:02:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.925258 9177 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.61:33187 every 8 connection(s)
07:02:31.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.926625 9107 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/data/info.pb
07:02:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.930186 9178 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.932957 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9107
07:02:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.933145 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-1/wal/instance
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.934160 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/logs
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data/info.pb
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.60:37981
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.60
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json with env {}
07:02:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.935986 9178 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } has no permanent_uuid. Determining permanent_uuid...
07:02:31.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.951294 9178 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } has no permanent_uuid. Determining permanent_uuid...
07:02:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:31.960060 9178 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } has no permanent_uuid. Determining permanent_uuid...
07:02:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.961184 9130 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:31.962834 9178 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.012138 9178 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 1
07:02:32.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.014106 9178 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.082325 9178 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 2
07:02:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.084304 9178 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.165032 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } attempt: 5
07:02:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.172102 9105 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } has no permanent_uuid. Determining permanent_uuid...
07:02:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.174077 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.175555 9178 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 3
07:02:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.177361 9178 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.225291 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 1
07:02:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.227056 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.271644 9180 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.271986 9180 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.272130 9180 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.279752 9180 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:02:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.279884 9180 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.279953 9180 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:02:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.280025 9180 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:02:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.280084 9180 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:02:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.280151 9180 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39113/jwks.json
07:02:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.289484 9180 master_runner.cc:386] Master server non-default flags:
07:02:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.60:37981
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39113/jwks.json
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data/info.pb
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.60
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/logs
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.291247 9180 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.293027 9180 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:32.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.295293 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 2
07:02:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.297102 9105 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.303264 9188 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.303905 9189 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.304577 9191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.305248 9180 server_base.cc:1034] running on GCE node
07:02:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.306125 9180 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.307595 9180 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.308826 9180 hybrid_clock.cc:648] HybridClock initialized: now 1714114952308765 us; error 34 us; skew 500 ppm
07:02:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.309170 9180 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.311515 9180 webserver.cc:460] Webserver started at http://127.8.209.60:35081/ using document root <none> and password file <none>
07:02:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.312230 9180 fs_manager.cc:359] Metadata directory not provided
07:02:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.312356 9180 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.312673 9180 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.315368 9180 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data/instance:
07:02:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f3da9d128c1346bf96d9b557a547958f"
07:02:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:32 on dist-test-slave-b8wt"
07:02:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.316176 9180 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal/instance:
07:02:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f3da9d128c1346bf96d9b557a547958f"
07:02:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:32 on dist-test-slave-b8wt"
07:02:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.320577 9178 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 4
07:02:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.321413 9180 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
07:02:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.322438 9178 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.8.209.60:37981: Network error: Client connection negotiation failed: client connection to 127.8.209.60:37981: connect: Connection refused (error 111)
07:02:32.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.325160 9197 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.326550 9180 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:02:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.326721 9180 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f3da9d128c1346bf96d9b557a547958f"
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:32 on dist-test-slave-b8wt"
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.326925 9180 fs_report.cc:383] FS layout report
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data/data
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.351068 9180 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.351985 9180 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.352311 9180 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:32.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.368553 9180 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.60:37981
07:02:32.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.368592 9248 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.60:37981 every 8 connection(s)
07:02:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.369953 9180 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/data/info.pb
07:02:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.373555 9249 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.377300 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9180
07:02:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.377452 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/master-2/wal/instance
07:02:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.379437 9249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } has no permanent_uuid. Determining permanent_uuid...
07:02:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.403314 9105 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 3
07:02:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.406996 9249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } has no permanent_uuid. Determining permanent_uuid...
07:02:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.414904 9249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } has no permanent_uuid. Determining permanent_uuid...
07:02:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.418995 9105 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: Bootstrap starting.
07:02:32.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.423710 9105 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: Neither blocks nor log segments found. Creating new log.
07:02:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.425488 9249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: Bootstrap starting.
07:02:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.425628 9105 log.cc:824] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: Log is configured to *not* fsync() on all Append() calls
07:02:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.428596 9105 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: No bootstrap required, opened a new log
07:02:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.429179 9249 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: Neither blocks nor log segments found. Creating new log.
07:02:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.430414 9249 log.cc:824] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: Log is configured to *not* fsync() on all Append() calls
07:02:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.433847 9249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: No bootstrap required, opened a new log
07:02:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.435976 9105 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.436228 9105 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.436304 9105 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c2b7184b67346f3acfeecc646a6c49f, State: Initialized, Role: FOLLOWER
07:02:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.436810 9105 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.438207 9258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.438575 9258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.439023 9105 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: configured and running, proceeding with master startup.
07:02:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.443029 9249 raft_consensus.cc:348] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.443504 9249 raft_consensus.cc:374] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.443645 9249 raft_consensus.cc:729] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f3da9d128c1346bf96d9b557a547958f, State: Initialized, Role: FOLLOWER
07:02:32.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.444459 9249 consensus_queue.cc:260] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.446080 9267 sys_catalog.cc:455] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.447368 9267 sys_catalog.cc:458] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.447403 9249 sys_catalog.cc:564] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: configured and running, proceeding with master startup.
07:02:32.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.452682 9270 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:02:32.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.452960 9270 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:02:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.457803 9059 tablet.cc:2369] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.451606 9039 master_runner.cc:186] Error getting master registration for 127.8.209.61:33187: OK, instance_id { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" instance_seqno: 1714114951919098 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:02:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.464309 9039 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:02:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.472831 9282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:02:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.473126 9282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:02:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.469843 9180 master_runner.cc:186] Error getting master registration for 127.8.209.61:33187: OK, instance_id { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" instance_seqno: 1714114951919098 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:02:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.479605 9180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:02:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.554950 9267 raft_consensus.cc:482] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.555269 9267 raft_consensus.cc:504] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.557154 9267 leader_election.cc:288] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1c2b7184b67346f3acfeecc646a6c49f (127.8.209.62:33719), 5352f3c1b7e643f28d6101dce3817a56 (127.8.209.61:33187)
07:02:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.559015 9080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f3da9d128c1346bf96d9b557a547958f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1c2b7184b67346f3acfeecc646a6c49f" is_pre_election: true
07:02:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.559542 9080 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f3da9d128c1346bf96d9b557a547958f in term 0.
07:02:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.560355 9200 leader_election.cc:302] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1c2b7184b67346f3acfeecc646a6c49f, f3da9d128c1346bf96d9b557a547958f; no voters:
07:02:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.560850 9267 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.561009 9267 raft_consensus.cc:482] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.561123 9267 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 0 FOLLOWER]: Advancing to term 1
07:02:32.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.564497 9267 raft_consensus.cc:504] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.565603 9080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f3da9d128c1346bf96d9b557a547958f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1c2b7184b67346f3acfeecc646a6c49f"
07:02:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.565819 9080 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 0 FOLLOWER]: Advancing to term 1
07:02:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.565907 9267 leader_election.cc:288] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [CANDIDATE]: Term 1 election: Requested vote from peers 1c2b7184b67346f3acfeecc646a6c49f (127.8.209.62:33719), 5352f3c1b7e643f28d6101dce3817a56 (127.8.209.61:33187)
07:02:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.569108 9080 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f3da9d128c1346bf96d9b557a547958f in term 1.
07:02:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.569535 9200 leader_election.cc:302] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1c2b7184b67346f3acfeecc646a6c49f, f3da9d128c1346bf96d9b557a547958f; no voters:
07:02:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.569895 9267 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 1 FOLLOWER]: Leader election won for term 1
07:02:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.570672 9267 raft_consensus.cc:686] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [term 1 LEADER]: Becoming Leader. State: Replica: f3da9d128c1346bf96d9b557a547958f, State: Running, Role: LEADER
07:02:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.571393 9267 consensus_queue.cc:237] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.575367 9284 sys_catalog.cc:455] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f3da9d128c1346bf96d9b557a547958f. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.575760 9284 sys_catalog.cc:458] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: This master's current role is: LEADER
07:02:32.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.587244 9286 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:02:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.589778 9286 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:02:32.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.601061 9080 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [term 1 FOLLOWER]: Refusing update from remote peer f3da9d128c1346bf96d9b557a547958f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:32.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.602059 9267 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:32.604537 9203 tablet.cc:2369] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:02:32.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.615510 9258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f3da9d128c1346bf96d9b557a547958f. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.615855 9258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.619508 9287 mvcc.cc:205] Tried to move back new op lower bound from 7021014845839323136 to 7021014845738577920. Current Snapshot: MvccSnapshot[applied={T|T < 7021014845839323136}]
07:02:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.621642 9178 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } attempt: 5
07:02:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.627415 9267 sys_catalog.cc:455] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.627728 9267 sys_catalog.cc:458] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: This master's current role is: LEADER
07:02:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.632498 9286 catalog_manager.cc:1347] Generated new cluster ID: 30d6e9febe8a467593b2db91820604f3
07:02:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.632616 9286 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:02:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.637333 9178 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: Bootstrap starting.
07:02:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.644843 9258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.645093 9258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.648727 9178 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: Neither blocks nor log segments found. Creating new log.
07:02:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.651453 9178 log.cc:824] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: Log is configured to *not* fsync() on all Append() calls
07:02:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.670079 9178 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: No bootstrap required, opened a new log
07:02:32.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.675693 9286 catalog_manager.cc:1370] Generated new certificate authority record
07:02:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.678872 9286 catalog_manager.cc:1504] Loading token signing keys...
07:02:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.694454 9178 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.695132 9178 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.695339 9178 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5352f3c1b7e643f28d6101dce3817a56, State: Initialized, Role: FOLLOWER
07:02:32.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.696699 9178 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } }
07:02:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.702474 9286 catalog_manager.cc:5857] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f: Generated new TSK 0
07:02:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.704406 9286 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:02:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.709278 9152 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f3da9d128c1346bf96d9b557a547958f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5352f3c1b7e643f28d6101dce3817a56"
07:02:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.709658 9151 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 0 FOLLOWER]: Advancing to term 1
07:02:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.713315 9151 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 1 FOLLOWER]: Refusing update from remote peer f3da9d128c1346bf96d9b557a547958f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.713840 9153 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f3da9d128c1346bf96d9b557a547958f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5352f3c1b7e643f28d6101dce3817a56" is_pre_election: true
07:02:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.714184 9153 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f3da9d128c1346bf96d9b557a547958f in term 1.
07:02:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.714238 9284 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [LEADER]: Connected to new peer: Peer: permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.723538 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/logs
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data/info.pb
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.1:0
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.1
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.1
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:02:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.730698 9178 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: configured and running, proceeding with master startup.
07:02:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.784654 9293 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.784883 9294 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f3da9d128c1346bf96d9b557a547958f. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.785171 9293 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.785254 9294 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.784718 9304 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.785625 9304 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56 [sys.catalog]: This master's current role is: FOLLOWER
07:02:32.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.834523 9311 catalog_manager.cc:1259] Loaded cluster ID: 30d6e9febe8a467593b2db91820604f3
07:02:32.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.834867 9311 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: loading cluster ID for follower catalog manager: success
07:02:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.843261 9311 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: acquiring CA information for follower catalog manager: success
07:02:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:32.846028 9311 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5352f3c1b7e643f28d6101dce3817a56: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:02:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.092362 9284 sys_catalog.cc:455] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f3da9d128c1346bf96d9b557a547958f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1c2b7184b67346f3acfeecc646a6c49f" member_type: VOTER last_known_addr { host: "127.8.209.62" port: 33719 } } peers { permanent_uuid: "5352f3c1b7e643f28d6101dce3817a56" member_type: VOTER last_known_addr { host: "127.8.209.61" port: 33187 } } peers { permanent_uuid: "f3da9d128c1346bf96d9b557a547958f" member_type: VOTER last_known_addr { host: "127.8.209.60" port: 37981 } } }
07:02:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.092684 9284 sys_catalog.cc:458] T 00000000000000000000000000000000 P f3da9d128c1346bf96d9b557a547958f [sys.catalog]: This master's current role is: LEADER
07:02:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.203279 9295 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.203570 9295 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.203651 9295 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:02:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.203719 9295 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.211395 9295 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.211537 9295 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:02:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.211588 9295 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:02:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.211660 9295 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.209.1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.220825 9295 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.1:0
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data/info.pb
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/logs
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:33.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.222301 9295 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:33.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.223915 9295 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.237514 9318 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.238204 9321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.238816 9319 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.239121 9295 server_base.cc:1034] running on GCE node
07:02:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.239733 9295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.240738 9295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:33.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.241938 9295 hybrid_clock.cc:648] HybridClock initialized: now 1714114953241861 us; error 72 us; skew 500 ppm
07:02:33.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.242317 9295 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.244688 9295 webserver.cc:460] Webserver started at http://127.8.209.1:35593/ using document root <none> and password file <none>
07:02:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.245398 9295 fs_manager.cc:359] Metadata directory not provided
07:02:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.245503 9295 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.245808 9295 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.248369 9295 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data/instance:
07:02:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a"
07:02:33.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.249162 9295 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal/instance:
07:02:33.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a"
07:02:33.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.254209 9295 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
07:02:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.257526 9327 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.258857 9295 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.258989 9295 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a"
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.259150 9295 fs_report.cc:383] FS layout report
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data/data
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.280272 9295 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:33.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.281067 9295 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:33.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.281306 9295 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.282347 9295 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:02:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.284049 9295 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.284139 9295 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.284219 9295 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.284263 9295 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:02:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.317704 9295 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.1:41725
07:02:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.317764 9439 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.1:41725 every 8 connection(s)
07:02:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.318957 9295 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/data/info.pb
07:02:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.321094 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9295
07:02:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.321323 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-0/wal/instance
07:02:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.327661 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/logs
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data/info.pb
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.2:0
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.2
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.2
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:02:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.360757 9441 heartbeater.cc:347] Connected to a master server at 127.8.209.60:37981
07:02:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.361155 9441 heartbeater.cc:464] Registering TS with master...
07:02:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.361279 9442 heartbeater.cc:347] Connected to a master server at 127.8.209.61:33187
07:02:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.361431 9442 heartbeater.cc:464] Registering TS with master...
07:02:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.361980 9440 heartbeater.cc:347] Connected to a master server at 127.8.209.62:33719
07:02:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.362123 9440 heartbeater.cc:464] Registering TS with master...
07:02:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.362588 9441 heartbeater.cc:511] Master 127.8.209.60:37981 requested a full tablet report, sending...
07:02:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.362635 9442 heartbeater.cc:511] Master 127.8.209.61:33187 requested a full tablet report, sending...
07:02:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.362866 9440 heartbeater.cc:511] Master 127.8.209.62:33719 requested a full tablet report, sending...
07:02:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.364936 9214 ts_manager.cc:194] Registered new tserver with Master: ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1:41725)
07:02:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.365002 9143 ts_manager.cc:194] Registered new tserver with Master: ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1:41725)
07:02:33.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.366556 9069 ts_manager.cc:194] Registered new tserver with Master: ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1:41725)
07:02:33.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.366967 9214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.209.1:57829
07:02:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.455010 9270 catalog_manager.cc:1259] Loaded cluster ID: 30d6e9febe8a467593b2db91820604f3
07:02:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.455102 9270 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: loading cluster ID for follower catalog manager: success
07:02:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.457551 9270 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: acquiring CA information for follower catalog manager: success
07:02:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.459230 9270 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1c2b7184b67346f3acfeecc646a6c49f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:02:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.676465 9450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.676766 9450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.676887 9450 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:02:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.676990 9450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.684391 9450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.684569 9450 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:02:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.684641 9450 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:02:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.684760 9450 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.209.2
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.693868 9450 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.2:0
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data/info.pb
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.2
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.2
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/logs
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:33.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:33.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:33.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:33.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.695446 9450 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:33.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.697224 9450 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.708715 9455 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.709292 9456 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:33.710026 9458 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.710647 9450 server_base.cc:1034] running on GCE node
07:02:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.711213 9450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.712078 9450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.713287 9450 hybrid_clock.cc:648] HybridClock initialized: now 1714114953713241 us; error 33 us; skew 500 ppm
07:02:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.713613 9450 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.715849 9450 webserver.cc:460] Webserver started at http://127.8.209.2:36219/ using document root <none> and password file <none>
07:02:33.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.716620 9450 fs_manager.cc:359] Metadata directory not provided
07:02:33.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.716728 9450 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.717036 9450 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.719628 9450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data/instance:
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "918ddcbd9d30478f8c35fedac4798b89"
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.720398 9450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal/instance:
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "918ddcbd9d30478f8c35fedac4798b89"
07:02:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.725498 9450 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
07:02:33.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.729187 9464 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.730533 9450 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:02:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.730700 9450 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "918ddcbd9d30478f8c35fedac4798b89"
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:33 on dist-test-slave-b8wt"
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.730938 9450 fs_report.cc:383] FS layout report
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data/data
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.756258 9450 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.757174 9450 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.757411 9450 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.758406 9450 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:02:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.759917 9450 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.760010 9450 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.760079 9450 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.760125 9450 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:02:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.792275 9450 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.2:39883
07:02:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.792336 9576 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.2:39883 every 8 connection(s)
07:02:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.793510 9450 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/data/info.pb
07:02:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.803758 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9450
07:02:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.803936 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-1/wal/instance
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.808089 9028 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/logs
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data/info.pb
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:02:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.3:0
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.3
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.3
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:02:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.810477 9578 heartbeater.cc:347] Connected to a master server at 127.8.209.60:37981
07:02:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.810608 9579 heartbeater.cc:347] Connected to a master server at 127.8.209.61:33187
07:02:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.810878 9578 heartbeater.cc:464] Registering TS with master...
07:02:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.810874 9579 heartbeater.cc:464] Registering TS with master...
07:02:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.812004 9577 heartbeater.cc:347] Connected to a master server at 127.8.209.62:33719
07:02:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.812096 9578 heartbeater.cc:511] Master 127.8.209.60:37981 requested a full tablet report, sending...
07:02:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.812147 9577 heartbeater.cc:464] Registering TS with master...
07:02:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.812155 9579 heartbeater.cc:511] Master 127.8.209.61:33187 requested a full tablet report, sending...
07:02:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.812721 9577 heartbeater.cc:511] Master 127.8.209.62:33719 requested a full tablet report, sending...
07:02:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.813711 9214 ts_manager.cc:194] Registered new tserver with Master: 918ddcbd9d30478f8c35fedac4798b89 (127.8.209.2:39883)
07:02:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.813750 9069 ts_manager.cc:194] Registered new tserver with Master: 918ddcbd9d30478f8c35fedac4798b89 (127.8.209.2:39883)
07:02:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.813817 9143 ts_manager.cc:194] Registered new tserver with Master: 918ddcbd9d30478f8c35fedac4798b89 (127.8.209.2:39883)
07:02:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:33.814952 9214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.209.2:38307
07:02:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.142817 9584 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:02:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.143111 9584 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:02:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.143194 9584 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:02:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.143261 9584 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:02:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.150862 9584 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:02:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.151008 9584 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:02:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.151057 9584 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:02:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.151129 9584 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.209.3
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.160389 9584 tablet_server_runner.cc:78] Tablet server non-default flags:
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.209.20:35103
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal
07:02:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.209.3:0
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data/info.pb
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.209.3
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.209.62:33719,127.8.209.61:33187,127.8.209.60:37981
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.209.3
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/logs
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:02:34.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.161687 9584 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:02:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.163436 9584 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:02:34.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.174978 9590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:34.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.175655 9592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:34.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.176287 9589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:02:34.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.176385 9584 server_base.cc:1034] running on GCE node
07:02:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.177014 9584 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:02:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.178016 9584 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:02:34.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.179222 9584 hybrid_clock.cc:648] HybridClock initialized: now 1714114954179173 us; error 40 us; skew 500 ppm
07:02:34.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.179544 9584 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:02:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.181784 9584 webserver.cc:460] Webserver started at http://127.8.209.3:40959/ using document root <none> and password file <none>
07:02:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.182550 9584 fs_manager.cc:359] Metadata directory not provided
07:02:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.182659 9584 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:02:34.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.182979 9584 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:02:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.185500 9584 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data/instance:
07:02:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "036f0b11d4394c499f14ad934647290b"
07:02:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:34 on dist-test-slave-b8wt"
07:02:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.186292 9584 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal/instance:
07:02:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "036f0b11d4394c499f14ad934647290b"
07:02:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:34 on dist-test-slave-b8wt"
07:02:34.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.191413 9584 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
07:02:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.194726 9598 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.196002 9584 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.196162 9584 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data,/tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "036f0b11d4394c499f14ad934647290b"
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:02:34 on dist-test-slave-b8wt"
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.196372 9584 fs_report.cc:383] FS layout report
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data/data
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:02:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:02:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:02:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.226151 9584 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:02:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.227006 9584 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:02:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.227243 9584 kserver.cc:163] Server-wide thread pool size limit: 3276
07:02:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.228205 9584 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:02:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.229804 9584 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:02:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.229892 9584 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:02:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.229990 9584 ts_tablet_manager.cc:590] Registered 0 tablets
07:02:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.230059 9584 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
07:02:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.260614 9584 rpc_server.cc:307] RPC server started. Bound to: 127.8.209.3:43753
07:02:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.260680 9711 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.209.3:43753 every 8 connection(s)
07:02:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.261922 9584 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/data/info.pb
07:02:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.269093 9028 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu as pid 9584
07:02:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.269304 9028 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBMRGjA/test-tmp/mini-kudu-cluster5959884480929023257/ts-2/wal/instance
07:02:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.280197 9712 heartbeater.cc:347] Connected to a master server at 127.8.209.62:33719
07:02:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.280218 9714 heartbeater.cc:347] Connected to a master server at 127.8.209.61:33187
07:02:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.280565 9714 heartbeater.cc:464] Registering TS with master...
07:02:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.281374 9713 heartbeater.cc:347] Connected to a master server at 127.8.209.60:37981
07:02:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.281530 9713 heartbeater.cc:464] Registering TS with master...
07:02:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.281559 9714 heartbeater.cc:511] Master 127.8.209.61:33187 requested a full tablet report, sending...
07:02:34.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.280568 9712 heartbeater.cc:464] Registering TS with master...
07:02:34.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.282112 9713 heartbeater.cc:511] Master 127.8.209.60:37981 requested a full tablet report, sending...
07:02:34.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.282624 9712 heartbeater.cc:511] Master 127.8.209.62:33719 requested a full tablet report, sending...
07:02:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.283337 9214 ts_manager.cc:194] Registered new tserver with Master: 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753)
07:02:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.283385 9143 ts_manager.cc:194] Registered new tserver with Master: 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753)
07:02:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.283758 9069 ts_manager.cc:194] Registered new tserver with Master: 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753)
07:02:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.284374 9214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.209.3:36583
07:02:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.294802 9028 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:02:34.296 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:02:34.296 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:02:34.297 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.209.62"
port: 33719
}
bound_http_address {
host: "127.8.209.62"
port: 36575
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.209.61"
port: 33187
}
bound_http_address {
host: "127.8.209.61"
port: 35031
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.209.60"
port: 37981
}
bound_http_address {
host: "127.8.209.60"
port: 35081
}
}
}
07:02:34.298 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:02:34.298 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.209.1"
port: 41725
}
bound_http_address {
host: "127.8.209.1"
port: 35593
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.209.2"
port: 39883
}
bound_http_address {
host: "127.8.209.2"
port: 36219
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.209.3"
port: 43753
}
bound_http_address {
host: "127.8.209.3"
port: 40959
}
}
}
07:02:34.299 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:02:34.300 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:02:34.301 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
07:02:34.302 [INFO - main] (Logging.scala:57) ==============================================================
07:02:34.302 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
07:02:34.302 [INFO - main] (Logging.scala:57) ==============================================================
07:02:34.302 [INFO - main] (Logging.scala:57) Submitted application: test
07:02:34.302 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
07:02:34.303 [INFO - main] (Logging.scala:57) Limiting resource is cpu
07:02:34.303 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
07:02:34.303 [INFO - main] (Logging.scala:57) Changing view acls to: slave
07:02:34.303 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
07:02:34.303 [INFO - main] (Logging.scala:57) Changing view acls groups to:
07:02:34.304 [INFO - main] (Logging.scala:57) Changing modify acls groups to:
07:02:34.304 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(slave); groups with view permissions: Set(); users with modify permissions: Set(slave); groups with modify permissions: Set()
07:02:34.332 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 34557.
07:02:34.334 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
07:02:34.335 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
07:02:34.335 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
07:02:34.335 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
07:02:34.335 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
07:02:34.336 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-e4db5e41-92b3-4e6b-8821-5750f084d635
07:02:34.336 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
07:02:34.337 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
07:02:34.358 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-b8wt.c.gcp-upstream.internal
07:02:34.361 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 44167.
07:02:34.361 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-b8wt.c.gcp-upstream.internal:44167
07:02:34.361 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
07:02:34.361 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-b8wt.c.gcp-upstream.internal, 44167, None)
07:02:34.361 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-b8wt.c.gcp-upstream.internal:44167 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-b8wt.c.gcp-upstream.internal, 44167, None)
07:02:34.362 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-b8wt.c.gcp-upstream.internal, 44167, None)
07:02:34.362 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-b8wt.c.gcp-upstream.internal, 44167, None)
07:02:34.367 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:02:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.371699 9441 heartbeater.cc:503] Master 127.8.209.60:37981 was elected leader, sending a full tablet report...
07:02:34.382 [DEBUG - main] (Connection.java:748) [peer master-127.8.209.62:33719(127.8.209.62:33719)] connecting to peer
07:02:34.383 [DEBUG - main] (Connection.java:748) [peer master-127.8.209.60:37981(127.8.209.60:37981)] connecting to peer
07:02:34.384 [DEBUG - main] (Connection.java:748) [peer master-127.8.209.61:33187(127.8.209.61:33187)] connecting to peer
07:02:34.385 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.209.60:37981(127.8.209.60:37981)] Successfully connected to peer
07:02:34.385 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.209.62:33719(127.8.209.62:33719)] Successfully connected to peer
07:02:34.386 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.209.60
07:02:34.387 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.209.61:33187(127.8.209.61:33187)] Successfully connected to peer
07:02:34.388 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.209.62
07:02:34.392 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.209.61
07:02:34.396 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:34.399 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:34.399 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc2afc076, L:/127.0.0.1:35174 - R:/127.8.209.60:37981] using SASL/PLAIN
07:02:34.401 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:02:34.402 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xeb8a442e, L:/127.0.0.1:47220 - R:/127.8.209.62:33719] using SASL/PLAIN
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.403693 9214 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35174:
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c1_i"
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c2_s"
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c3_double"
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DOUBLE
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c4_long"
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c5_bool"
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c6_short"
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT16
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c7_float"
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: FLOAT
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c8_binary"
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BINARY
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c9_unixtime_micros"
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: UNIXTIME_MICROS
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c10_byte"
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.407 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c11_decimal32"
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL32
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 9
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c12_decimal64"
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL64
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 18
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c13_decimal128"
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL128
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 38
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c14_varchar"
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: VARCHAR
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) length: 65535
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c15_date"
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DATE
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
07:02:34.410 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb8e646f0, L:/127.0.0.1:43418 - R:/127.8.209.61:33187] using SASL/PLAIN
07:02:34.445 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table efb838cffbcf4cfdb7ae36996ecfdc81
07:02:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.461057 9647 tablet_service.cc:1467] Processing CreateTablet for tablet 3988a138844e4d1aad1d5ce02f3c03f1 (DEFAULT_TABLE table=test [id=efb838cffbcf4cfdb7ae36996ecfdc81]), partition=RANGE (key) PARTITION VALUES < 50
07:02:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.461398 9375 tablet_service.cc:1467] Processing CreateTablet for tablet a08a99b13a484d53b2d1fe01acc7748a (DEFAULT_TABLE table=test [id=efb838cffbcf4cfdb7ae36996ecfdc81]), partition=RANGE (key) PARTITION 50 <= VALUES
07:02:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.462770 9375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a08a99b13a484d53b2d1fe01acc7748a. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.462800 9647 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3988a138844e4d1aad1d5ce02f3c03f1. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.472492 9765 tablet_bootstrap.cc:492] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: Bootstrap starting.
07:02:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.472584 9766 tablet_bootstrap.cc:492] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: Bootstrap starting.
07:02:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.476032 9766 tablet_bootstrap.cc:654] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: Neither blocks nor log segments found. Creating new log.
07:02:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.476089 9765 tablet_bootstrap.cc:654] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: Neither blocks nor log segments found. Creating new log.
07:02:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.477049 9766 log.cc:824] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: Log is configured to *not* fsync() on all Append() calls
07:02:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.477221 9765 log.cc:824] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: Log is configured to *not* fsync() on all Append() calls
07:02:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.479748 9765 tablet_bootstrap.cc:492] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: No bootstrap required, opened a new log
07:02:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.480000 9766 tablet_bootstrap.cc:492] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: No bootstrap required, opened a new log
07:02:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.480015 9765 ts_tablet_manager.cc:1367] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
07:02:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.480232 9766 ts_tablet_manager.cc:1367] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.000s
07:02:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.486410 9765 raft_consensus.cc:348] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } }
07:02:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.486743 9765 raft_consensus.cc:374] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.486846 9765 raft_consensus.cc:729] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ccaf3b1f4b9e4673b9b942ff3becee0a, State: Initialized, Role: FOLLOWER
07:02:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.487464 9765 consensus_queue.cc:260] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } }
07:02:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.487805 9765 raft_consensus.cc:388] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:02:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.487946 9765 raft_consensus.cc:482] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:02:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.488121 9765 raft_consensus.cc:3037] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Advancing to term 1
07:02:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.489019 9766 raft_consensus.cc:348] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.489385 9766 raft_consensus.cc:374] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.489475 9766 raft_consensus.cc:729] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 036f0b11d4394c499f14ad934647290b, State: Initialized, Role: FOLLOWER
07:02:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490114 9766 consensus_queue.cc:260] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490337 9766 raft_consensus.cc:388] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:02:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490324 9765 raft_consensus.cc:504] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } }
07:02:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490456 9766 raft_consensus.cc:482] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:02:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490595 9766 raft_consensus.cc:3037] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Advancing to term 1
07:02:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.490881 9765 leader_election.cc:302] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ccaf3b1f4b9e4673b9b942ff3becee0a; no voters:
07:02:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.491535 9765 leader_election.cc:288] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [CANDIDATE]: Term 1 election: Requested vote from peers
07:02:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.491634 9769 raft_consensus.cc:2781] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 1 FOLLOWER]: Leader election won for term 1
07:02:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.493571 9766 raft_consensus.cc:504] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.494220 9766 leader_election.cc:302] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 036f0b11d4394c499f14ad934647290b; no voters:
07:02:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.496057 9769 raft_consensus.cc:686] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [term 1 LEADER]: Becoming Leader. State: Replica: ccaf3b1f4b9e4673b9b942ff3becee0a, State: Running, Role: LEADER
07:02:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.496909 9769 consensus_queue.cc:237] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } }
07:02:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.500900 9765 ts_tablet_manager.cc:1398] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a: Time spent starting tablet: real 0.021s user 0.012s sys 0.008s
07:02:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.500985 9772 raft_consensus.cc:2781] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 1 FOLLOWER]: Leader election won for term 1
07:02:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.501391 9766 leader_election.cc:288] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [CANDIDATE]: Term 1 election: Requested vote from peers
07:02:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.501690 9772 raft_consensus.cc:686] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [term 1 LEADER]: Becoming Leader. State: Replica: 036f0b11d4394c499f14ad934647290b, State: Running, Role: LEADER
07:02:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.502563 9713 heartbeater.cc:503] Master 127.8.209.60:37981 was elected leader, sending a full tablet report...
07:02:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.502471 9772 consensus_queue.cc:237] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.509739 9214 catalog_manager.cc:5484] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b reported cstate change: term changed from 0 to 1, leader changed from <none> to 036f0b11d4394c499f14ad934647290b (127.8.209.3). New cstate: current_term: 1 leader_uuid: "036f0b11d4394c499f14ad934647290b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } health_report { overall_health: HEALTHY } } }
07:02:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.515658 9213 catalog_manager.cc:5484] T a08a99b13a484d53b2d1fe01acc7748a P ccaf3b1f4b9e4673b9b942ff3becee0a reported cstate change: term changed from 0 to 1, leader changed from <none> to ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1). New cstate: current_term: 1 leader_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } health_report { overall_health: HEALTHY } } }
07:02:34.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.533146 9766 ts_tablet_manager.cc:1398] T 3988a138844e4d1aad1d5ce02f3c03f1 P 036f0b11d4394c499f14ad934647290b: Time spent starting tablet: real 0.053s user 0.027s sys 0.024s
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.565053 9213 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35174:
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "val"
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.572 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 4112fad7b0c74d03a67b977a3ac11153
07:02:34.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.574716 9443 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:a08a99b13a484d53b2d1fe01acc7748a)
07:02:34.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.585072 9512 tablet_service.cc:1467] Processing CreateTablet for tablet b7d5776b50e9499b8b3dd11123ced5c9 (DEFAULT_TABLE table=simple-test [id=4112fad7b0c74d03a67b977a3ac11153]), partition=RANGE (key) PARTITION UNBOUNDED
07:02:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.586295 9512 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b7d5776b50e9499b8b3dd11123ced5c9. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.595520 9781 tablet_bootstrap.cc:492] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: Bootstrap starting.
07:02:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.598161 9781 tablet_bootstrap.cc:654] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: Neither blocks nor log segments found. Creating new log.
07:02:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.599141 9781 log.cc:824] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: Log is configured to *not* fsync() on all Append() calls
07:02:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.601236 9781 tablet_bootstrap.cc:492] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: No bootstrap required, opened a new log
07:02:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.601490 9781 ts_tablet_manager.cc:1367] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.004s
07:02:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.607605 9781 raft_consensus.cc:348] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } }
07:02:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.607939 9781 raft_consensus.cc:374] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.608037 9781 raft_consensus.cc:729] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 918ddcbd9d30478f8c35fedac4798b89, State: Initialized, Role: FOLLOWER
07:02:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.608595 9781 consensus_queue.cc:260] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } }
07:02:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.608808 9781 raft_consensus.cc:388] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:02:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.608955 9781 raft_consensus.cc:482] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:02:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.609122 9781 raft_consensus.cc:3037] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Advancing to term 1
07:02:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.611112 9781 raft_consensus.cc:504] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } }
07:02:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.611593 9781 leader_election.cc:302] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 918ddcbd9d30478f8c35fedac4798b89; no voters:
07:02:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.612246 9781 leader_election.cc:288] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 election: Requested vote from peers
07:02:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.612373 9783 raft_consensus.cc:2781] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 1 FOLLOWER]: Leader election won for term 1
07:02:34.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.613368 9783 raft_consensus.cc:686] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [term 1 LEADER]: Becoming Leader. State: Replica: 918ddcbd9d30478f8c35fedac4798b89, State: Running, Role: LEADER
07:02:34.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.613658 9578 heartbeater.cc:503] Master 127.8.209.60:37981 was elected leader, sending a full tablet report...
07:02:34.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.614043 9783 consensus_queue.cc:237] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } }
07:02:34.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.614214 9781 ts_tablet_manager.cc:1398] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89: Time spent starting tablet: real 0.013s user 0.013s sys 0.001s
07:02:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.619170 9213 catalog_manager.cc:5484] T b7d5776b50e9499b8b3dd11123ced5c9 P 918ddcbd9d30478f8c35fedac4798b89 reported cstate change: term changed from 0 to 1, leader changed from <none> to 918ddcbd9d30478f8c35fedac4798b89 (127.8.209.2). New cstate: current_term: 1 leader_uuid: "918ddcbd9d30478f8c35fedac4798b89" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } health_report { overall_health: HEALTHY } } }
07:02:34.643 [INFO - main] (RandomUtils.java:49) Using random seed: 1714114954643
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.644973 9213 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35174:
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "split-size-table"
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c1_i"
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c2_s"
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c3_double"
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DOUBLE
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c4_long"
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c5_bool"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c6_short"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT16
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c7_float"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: FLOAT
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c8_binary"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BINARY
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c9_unixtime_micros"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: UNIXTIME_MICROS
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c10_byte"
07:02:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c11_decimal32"
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL32
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 9
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c12_decimal64"
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL64
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 18
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c13_decimal128"
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL128
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 38
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c14_varchar"
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: VARCHAR
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) length: 65535
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c15_date"
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DATE
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:02:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.646082 9213 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table split-size-table in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:02:34.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 645a3ddf59634af1b28990c33346a513
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.657371 9375 tablet_service.cc:1467] Processing CreateTablet for tablet 5495564ce7a540c3b9ebc2340735f3ff (DEFAULT_TABLE table=split-size-table [id=645a3ddf59634af1b28990c33346a513]), partition=RANGE (key) PARTITION UNBOUNDED
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.657866 9375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5495564ce7a540c3b9ebc2340735f3ff. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.657907 9512 tablet_service.cc:1467] Processing CreateTablet for tablet 5495564ce7a540c3b9ebc2340735f3ff (DEFAULT_TABLE table=split-size-table [id=645a3ddf59634af1b28990c33346a513]), partition=RANGE (key) PARTITION UNBOUNDED
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.657984 9647 tablet_service.cc:1467] Processing CreateTablet for tablet 5495564ce7a540c3b9ebc2340735f3ff (DEFAULT_TABLE table=split-size-table [id=645a3ddf59634af1b28990c33346a513]), partition=RANGE (key) PARTITION UNBOUNDED
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.658339 9647 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5495564ce7a540c3b9ebc2340735f3ff. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.658337 9512 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5495564ce7a540c3b9ebc2340735f3ff. 1 dirs total, 0 dirs full, 0 dirs failed
07:02:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.662285 9765 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a: Bootstrap starting.
07:02:34.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.663724 9765 tablet_bootstrap.cc:654] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a: Neither blocks nor log segments found. Creating new log.
07:02:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.664746 9781 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89: Bootstrap starting.
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.665570 9766 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b: Bootstrap starting.
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.666301 9781 tablet_bootstrap.cc:654] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89: Neither blocks nor log segments found. Creating new log.
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.666436 9765 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a: No bootstrap required, opened a new log
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.666579 9765 ts_tablet_manager.cc:1367] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.667215 9766 tablet_bootstrap.cc:654] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b: Neither blocks nor log segments found. Creating new log.
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.667541 9765 raft_consensus.cc:348] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.667778 9765 raft_consensus.cc:374] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.667867 9765 raft_consensus.cc:729] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ccaf3b1f4b9e4673b9b942ff3becee0a, State: Initialized, Role: FOLLOWER
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.668200 9781 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89: No bootstrap required, opened a new log
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.668102 9765 consensus_queue.cc:260] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.668301 9781 ts_tablet_manager.cc:1367] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
07:02:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.668722 9765 ts_tablet_manager.cc:1398] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
07:02:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.669170 9781 raft_consensus.cc:348] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.669385 9781 raft_consensus.cc:374] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.669479 9781 raft_consensus.cc:729] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 918ddcbd9d30478f8c35fedac4798b89, State: Initialized, Role: FOLLOWER
07:02:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.669740 9781 consensus_queue.cc:260] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.670607 9781 ts_tablet_manager.cc:1398] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
07:02:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.672307 9766 tablet_bootstrap.cc:492] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b: No bootstrap required, opened a new log
07:02:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.672573 9766 ts_tablet_manager.cc:1367] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
07:02:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.673738 9766 raft_consensus.cc:348] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.674127 9766 raft_consensus.cc:374] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:02:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.674332 9766 raft_consensus.cc:729] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 036f0b11d4394c499f14ad934647290b, State: Initialized, Role: FOLLOWER
07:02:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.674731 9766 consensus_queue.cc:260] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.676388 9766 ts_tablet_manager.cc:1398] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
07:02:34.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.735728 9783 raft_consensus.cc:482] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:02:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.735962 9783 raft_consensus.cc:504] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.737309 9783 leader_election.cc:288] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1:41725), 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753)
07:02:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.746079 9667 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5495564ce7a540c3b9ebc2340735f3ff" candidate_uuid: "918ddcbd9d30478f8c35fedac4798b89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "036f0b11d4394c499f14ad934647290b" is_pre_election: true
07:02:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.746435 9667 raft_consensus.cc:2454] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 918ddcbd9d30478f8c35fedac4798b89 in term 0.
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747012 9468 leader_election.cc:302] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 036f0b11d4394c499f14ad934647290b, 918ddcbd9d30478f8c35fedac4798b89; no voters:
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747294 9783 raft_consensus.cc:2781] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747373 9783 raft_consensus.cc:482] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747448 9783 raft_consensus.cc:3037] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 0 FOLLOWER]: Advancing to term 1
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747363 9395 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5495564ce7a540c3b9ebc2340735f3ff" candidate_uuid: "918ddcbd9d30478f8c35fedac4798b89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" is_pre_election: true
07:02:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.747840 9395 raft_consensus.cc:2454] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 918ddcbd9d30478f8c35fedac4798b89 in term 0.
07:02:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.749540 9783 raft_consensus.cc:504] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.750161 9783 leader_election.cc:288] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 election: Requested vote from peers ccaf3b1f4b9e4673b9b942ff3becee0a (127.8.209.1:41725), 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753)
07:02:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.750474 9395 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5495564ce7a540c3b9ebc2340735f3ff" candidate_uuid: "918ddcbd9d30478f8c35fedac4798b89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a"
07:02:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.750563 9667 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5495564ce7a540c3b9ebc2340735f3ff" candidate_uuid: "918ddcbd9d30478f8c35fedac4798b89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "036f0b11d4394c499f14ad934647290b"
07:02:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.750646 9395 raft_consensus.cc:3037] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 0 FOLLOWER]: Advancing to term 1
07:02:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.750726 9667 raft_consensus.cc:3037] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 0 FOLLOWER]: Advancing to term 1
07:02:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.752645 9395 raft_consensus.cc:2454] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 918ddcbd9d30478f8c35fedac4798b89 in term 1.
07:02:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.752642 9667 raft_consensus.cc:2454] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 918ddcbd9d30478f8c35fedac4798b89 in term 1.
07:02:34.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.753021 9468 leader_election.cc:302] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 036f0b11d4394c499f14ad934647290b, 918ddcbd9d30478f8c35fedac4798b89; no voters:
07:02:34.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.753283 9783 raft_consensus.cc:2781] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 1 FOLLOWER]: Leader election won for term 1
07:02:34.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.753438 9783 raft_consensus.cc:686] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [term 1 LEADER]: Becoming Leader. State: Replica: 918ddcbd9d30478f8c35fedac4798b89, State: Running, Role: LEADER
07:02:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.753746 9783 consensus_queue.cc:237] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } }
07:02:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.757202 9213 catalog_manager.cc:5484] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 reported cstate change: term changed from 0 to 1, leader changed from <none> to 918ddcbd9d30478f8c35fedac4798b89 (127.8.209.2). New cstate: current_term: 1 leader_uuid: "918ddcbd9d30478f8c35fedac4798b89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "918ddcbd9d30478f8c35fedac4798b89" member_type: VOTER last_known_addr { host: "127.8.209.2" port: 39883 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 } health_report { overall_health: UNKNOWN } } }
07:02:34.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.766300 9715 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:3988a138844e4d1aad1d5ce02f3c03f1)
07:02:34.772 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:34.772 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:34.772 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [2ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=null) after lookup
07:02:34.773 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:02:34.774 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] Successfully connected to peer
07:02:34.782 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:34.784 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x867004a8, L:/127.0.0.1:35882 - R:/127.8.209.2:39883] using TOKEN/null
07:02:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.793534 9667 raft_consensus.cc:1261] T 5495564ce7a540c3b9ebc2340735f3ff P 036f0b11d4394c499f14ad934647290b [term 1 FOLLOWER]: Refusing update from remote peer 918ddcbd9d30478f8c35fedac4798b89: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.793534 9395 raft_consensus.cc:1261] T 5495564ce7a540c3b9ebc2340735f3ff P ccaf3b1f4b9e4673b9b942ff3becee0a [term 1 FOLLOWER]: Refusing update from remote peer 918ddcbd9d30478f8c35fedac4798b89: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:02:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.794574 9784 consensus_queue.cc:1035] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "036f0b11d4394c499f14ad934647290b" member_type: VOTER last_known_addr { host: "127.8.209.3" port: 43753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.794890 9783 consensus_queue.cc:1035] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ccaf3b1f4b9e4673b9b942ff3becee0a" member_type: VOTER last_known_addr { host: "127.8.209.1" port: 41725 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:02:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:34.796792 9580 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:5495564ce7a540c3b9ebc2340735f3ff)
07:02:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:34.807334 9787 mvcc.cc:205] Tried to move back new op lower bound from 7021014854823809024 to 7021014854675464192. Current Snapshot: MvccSnapshot[applied={T|T < 7021014854823809024}]
07:02:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.018020 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.049178 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.076861 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.233232 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.214s user 0.054s sys 0.012s Metrics: {"bytes_written":39425,"cfile_init":1,"compiler_manager_pool.queue_time_us":442,"dirs.queue_time_us":9300,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":7810,"drs_written":1,"lbm_read_time_us":98,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2336,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28,"thread_start_us":9739,"threads_started":2}
07:02:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.251417 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 446 bytes on disk
07:02:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.252426 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":83,"lbm_reads_lt_1ms":4}
07:02:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:35.366098 9198 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.8.209.61:33187, user_credentials={real_user=slave}} blocked reactor thread for 62397.8us
07:02:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.386049 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.288s user 0.064s sys 0.012s Metrics: {"bytes_written":39640,"cfile_init":1,"compiler_manager_pool.queue_time_us":12196,"dirs.queue_time_us":1161,"dirs.run_cpu_time_us":196,"dirs.run_wall_time_us":6515,"drs_written":1,"lbm_read_time_us":98,"lbm_reads_lt_1ms":4,"lbm_write_time_us":9646,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":33,"thread_start_us":13400,"threads_started":2}
07:02:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.388036 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 507 bytes on disk
07:02:35.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.389046 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":96,"lbm_reads_lt_1ms":4}
07:02:35.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.508773 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.519359 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.469s user 0.066s sys 0.014s Metrics: {"bytes_written":39440,"cfile_init":1,"compiler_manager_pool.queue_time_us":490,"dirs.queue_time_us":79080,"dirs.run_cpu_time_us":236,"dirs.run_wall_time_us":25275,"drs_written":1,"lbm_read_time_us":110,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10656,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":81259,"threads_started":2}
07:02:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.533329 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.598014 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.089s user 0.041s sys 0.000s Metrics: {"bytes_written":39713,"cfile_init":1,"dirs.queue_time_us":44,"dirs.run_cpu_time_us":191,"dirs.run_wall_time_us":1503,"drs_written":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4,"lbm_write_time_us":14477,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":38}
07:02:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.598835 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 562 bytes on disk
07:02:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.599553 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4}
07:02:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.642341 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.684023 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.144s user 0.045s sys 0.002s Metrics: {"bytes_written":39731,"cfile_init":1,"dirs.queue_time_us":59,"dirs.run_cpu_time_us":196,"dirs.run_wall_time_us":3087,"drs_written":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4,"lbm_write_time_us":16489,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"spinlock_wait_cycles":1280}
07:02:35.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.684758 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 1033 bytes on disk
07:02:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.685943 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":2,"lbm_read_time_us":145,"lbm_reads_lt_1ms":8}
07:02:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.734009 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.091s user 0.041s sys 0.004s Metrics: {"bytes_written":39810,"cfile_init":1,"dirs.queue_time_us":14113,"dirs.run_cpu_time_us":214,"dirs.run_wall_time_us":1447,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2471,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:35.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.734831 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 596 bytes on disk
07:02:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.739470 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4}
07:02:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.938243 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:35.990670 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.039592 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.101s user 0.038s sys 0.004s Metrics: {"bytes_written":39712,"cfile_init":1,"dirs.queue_time_us":44,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1442,"drs_written":1,"lbm_read_time_us":70,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2313,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33}
07:02:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.040637 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 506 bytes on disk
07:02:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.041389 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":92,"lbm_reads_lt_1ms":4}
07:02:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.069186 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.078s user 0.036s sys 0.004s Metrics: {"bytes_written":39564,"cfile_init":1,"dirs.queue_time_us":1747,"dirs.run_cpu_time_us":187,"dirs.run_wall_time_us":1518,"drs_written":1,"lbm_read_time_us":75,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2470,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31}
07:02:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.073987 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 481 bytes on disk
07:02:36.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.074934 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":91,"lbm_reads_lt_1ms":4}
07:02:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.101859 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.331794 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.229s user 0.049s sys 0.008s Metrics: {"bytes_written":39797,"cfile_init":1,"dirs.queue_time_us":11383,"dirs.run_cpu_time_us":206,"dirs.run_wall_time_us":7283,"drs_written":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2481,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":47,"thread_start_us":11522,"threads_started":1}
07:02:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.336568 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 661 bytes on disk
07:02:36.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.337504 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":93,"lbm_reads_lt_1ms":4}
07:02:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.543493 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.576428 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.589128 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.645347 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.101s user 0.053s sys 0.008s Metrics: {"bytes_written":39912,"cfile_init":1,"dirs.queue_time_us":19762,"dirs.run_cpu_time_us":198,"dirs.run_wall_time_us":1586,"drs_written":1,"lbm_read_time_us":76,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3198,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":45,"thread_start_us":19602,"threads_started":1}
07:02:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.646363 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 642 bytes on disk
07:02:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.647157 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4}
07:02:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.699194 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.110s user 0.033s sys 0.008s Metrics: {"bytes_written":39788,"cfile_init":1,"compiler_manager_pool.queue_time_us":92851,"compiler_manager_pool.run_cpu_time_us":84,"compiler_manager_pool.run_wall_time_us":84,"dirs.queue_time_us":194,"dirs.run_cpu_time_us":221,"dirs.run_wall_time_us":1459,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2129,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36}
07:02:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.700124 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 537 bytes on disk
07:02:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.701948 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4}
07:02:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.702960 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.126s user 0.038s sys 0.013s Metrics: {"bytes_written":39925,"cfile_init":1,"dirs.queue_time_us":4385,"dirs.run_cpu_time_us":209,"dirs.run_wall_time_us":1050,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3154,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43,"thread_start_us":4262,"threads_started":1}
07:02:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.703740 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 617 bytes on disk
07:02:36.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.704824 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":88,"lbm_reads_lt_1ms":4}
07:02:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.902070 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.940629 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.038s user 0.027s sys 0.008s Metrics: {"bytes_written":39776,"cfile_init":1,"dirs.queue_time_us":42,"dirs.run_cpu_time_us":228,"dirs.run_wall_time_us":1639,"drs_written":1,"lbm_read_time_us":81,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1823,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:36.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.941241 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 560 bytes on disk
07:02:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.942091 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
07:02:36.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.953127 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:36.955904 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.008196 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.052s user 0.042s sys 0.004s Metrics: {"bytes_written":39891,"cfile_init":1,"dirs.queue_time_us":65,"dirs.run_cpu_time_us":177,"dirs.run_wall_time_us":1421,"drs_written":1,"lbm_read_time_us":76,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3276,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":41}
07:02:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.008909 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 596 bytes on disk
07:02:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.010741 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4}
07:02:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.013156 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.060s user 0.037s sys 0.008s Metrics: {"bytes_written":39775,"cfile_init":1,"dirs.queue_time_us":121,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":1244,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2313,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.013765 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.014292 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
07:02:37.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.193001 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.232353 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.039s user 0.036s sys 0.000s Metrics: {"bytes_written":39809,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1470,"drs_written":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2123,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.233084 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 552 bytes on disk
07:02:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.233688 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
07:02:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.265041 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.265122 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.315625 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.050s user 0.032s sys 0.007s Metrics: {"bytes_written":39887,"cfile_init":1,"dirs.queue_time_us":755,"dirs.run_cpu_time_us":185,"dirs.run_wall_time_us":1447,"drs_written":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2623,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:37.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.316435 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 597 bytes on disk
07:02:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.317325 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
07:02:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.323165 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.058s user 0.036s sys 0.010s Metrics: {"bytes_written":39887,"cfile_init":1,"dirs.queue_time_us":42,"dirs.run_cpu_time_us":138,"dirs.run_wall_time_us":1833,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2208,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.324456 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 597 bytes on disk
07:02:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.325263 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":84,"lbm_reads_lt_1ms":4}
07:02:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.484637 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.524106 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.039s user 0.036s sys 0.000s Metrics: {"bytes_written":39754,"cfile_init":1,"dirs.queue_time_us":52,"dirs.run_cpu_time_us":163,"dirs.run_wall_time_us":1410,"drs_written":1,"lbm_read_time_us":70,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2059,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":35}
07:02:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.525075 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 526 bytes on disk
07:02:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.525662 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4}
07:02:37.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.569020 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.576333 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.606854 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.038s user 0.033s sys 0.004s Metrics: {"bytes_written":39764,"cfile_init":1,"dirs.queue_time_us":59,"dirs.run_cpu_time_us":177,"dirs.run_wall_time_us":1758,"drs_written":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2077,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.608350 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.609015 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":80,"lbm_reads_lt_1ms":4}
07:02:37.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.621279 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.045s user 0.034s sys 0.002s Metrics: {"bytes_written":39758,"cfile_init":1,"dirs.queue_time_us":58,"dirs.run_cpu_time_us":211,"dirs.run_wall_time_us":1526,"drs_written":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2160,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.622087 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.622893 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":83,"lbm_reads_lt_1ms":4}
07:02:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.776919 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.822856 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.039s sys 0.000s Metrics: {"bytes_written":39827,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":182,"dirs.run_wall_time_us":1585,"drs_written":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1858,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.823596 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:37.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.824430 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":70,"lbm_reads_lt_1ms":4}
07:02:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.859982 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.874032 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.907487 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.047s user 0.037s sys 0.000s Metrics: {"bytes_written":39816,"cfile_init":1,"dirs.queue_time_us":79,"dirs.run_cpu_time_us":224,"dirs.run_wall_time_us":1510,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2206,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36}
07:02:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.908326 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 536 bytes on disk
07:02:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.909106 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
07:02:37.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.920569 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.038s sys 0.000s Metrics: {"bytes_written":39783,"cfile_init":1,"dirs.queue_time_us":48,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1895,"drs_written":1,"lbm_read_time_us":94,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2813,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:37.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.921402 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:37.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:37.922227 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":90,"lbm_reads_lt_1ms":4}
07:02:38.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.075409 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.118528 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.043s user 0.037s sys 0.000s Metrics: {"bytes_written":39839,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":178,"dirs.run_wall_time_us":1485,"drs_written":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1883,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":35}
07:02:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.119201 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 526 bytes on disk
07:02:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.120261 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":75,"lbm_reads_lt_1ms":4}
07:02:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.160887 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.173226 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.205401 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.044s user 0.037s sys 0.003s Metrics: {"bytes_written":39848,"cfile_init":1,"dirs.queue_time_us":69,"dirs.run_cpu_time_us":202,"dirs.run_wall_time_us":1845,"drs_written":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2075,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.206465 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.208370 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":81,"lbm_reads_lt_1ms":4}
07:02:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.223124 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.049s user 0.040s sys 0.003s Metrics: {"bytes_written":39844,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":197,"dirs.run_wall_time_us":1954,"drs_written":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2095,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.223876 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.224617 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":94,"lbm_reads_lt_1ms":4}
07:02:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.371317 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.419731 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.042s user 0.033s sys 0.007s Metrics: {"bytes_written":39763,"cfile_init":1,"dirs.queue_time_us":59,"dirs.run_cpu_time_us":146,"dirs.run_wall_time_us":2310,"drs_written":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1951,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:02:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.420658 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 571 bytes on disk
07:02:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.421262 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
07:02:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.459513 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.475652 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.509527 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.050s user 0.041s sys 0.001s Metrics: {"bytes_written":39828,"cfile_init":1,"dirs.queue_time_us":47,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":1379,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2232,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.511160 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:38.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.512084 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":84,"lbm_reads_lt_1ms":4}
07:02:38.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.521003 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.045s user 0.040s sys 0.002s Metrics: {"bytes_written":39830,"cfile_init":1,"dirs.queue_time_us":63,"dirs.run_cpu_time_us":202,"dirs.run_wall_time_us":3058,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2184,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.521832 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.522867 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":98,"lbm_reads_lt_1ms":4}
07:02:38.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.672166 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.712024 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.040s user 0.036s sys 0.000s Metrics: {"bytes_written":39810,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":188,"dirs.run_wall_time_us":1487,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2240,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34}
07:02:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.712817 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 516 bytes on disk
07:02:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.713585 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
07:02:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.763638 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.774081 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.809417 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.042s sys 0.000s Metrics: {"bytes_written":39856,"cfile_init":1,"dirs.queue_time_us":111,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":2078,"drs_written":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2049,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37,"spinlock_wait_cycles":5376}
07:02:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.810207 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.817178 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4}
07:02:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.830883 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.056s user 0.040s sys 0.005s Metrics: {"bytes_written":39840,"cfile_init":1,"dirs.queue_time_us":139,"dirs.run_cpu_time_us":184,"dirs.run_wall_time_us":1367,"drs_written":1,"lbm_read_time_us":75,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2111,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.831665 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.832484 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":100,"lbm_reads_lt_1ms":4}
07:02:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:38.964871 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.003901 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.039s user 0.036s sys 0.000s Metrics: {"bytes_written":39848,"cfile_init":1,"dirs.queue_time_us":63,"dirs.run_cpu_time_us":188,"dirs.run_wall_time_us":1512,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1776,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37}
07:02:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.004621 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 551 bytes on disk
07:02:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.005517 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":88,"lbm_reads_lt_1ms":4}
07:02:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.068199 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.083549 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.117229 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.049s user 0.038s sys 0.000s Metrics: {"bytes_written":39859,"cfile_init":1,"dirs.queue_time_us":50,"dirs.run_cpu_time_us":149,"dirs.run_wall_time_us":1543,"drs_written":1,"lbm_read_time_us":73,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2428,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.117973 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.118659 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":71,"lbm_reads_lt_1ms":4}
07:02:39.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.134771 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.051s user 0.045s sys 0.000s Metrics: {"bytes_written":39830,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":180,"dirs.run_wall_time_us":1967,"drs_written":1,"lbm_read_time_us":92,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2243,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:39.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.135571 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.136473 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4}
07:02:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.256587 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.303206 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.039s sys 0.000s Metrics: {"bytes_written":39760,"cfile_init":1,"dirs.queue_time_us":56,"dirs.run_cpu_time_us":170,"dirs.run_wall_time_us":1738,"drs_written":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1929,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.303915 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.304575 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4,"spinlock_wait_cycles":23296}
07:02:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.369719 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.387414 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.425804 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.056s user 0.044s sys 0.007s Metrics: {"bytes_written":39725,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1449,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2166,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.427069 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.428020 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":73,"lbm_reads_lt_1ms":4}
07:02:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.457278 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.070s user 0.047s sys 0.008s Metrics: {"bytes_written":39723,"cfile_init":1,"dirs.queue_time_us":48,"dirs.run_cpu_time_us":194,"dirs.run_wall_time_us":1418,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2312,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.458261 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.458984 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4}
07:02:39.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.555549 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.593109 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.037s user 0.029s sys 0.004s Metrics: {"bytes_written":39817,"cfile_init":1,"dirs.queue_time_us":71,"dirs.run_cpu_time_us":209,"dirs.run_wall_time_us":1561,"drs_written":1,"lbm_read_time_us":83,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1650,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38,"spinlock_wait_cycles":896}
07:02:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.594202 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.595057 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":86,"lbm_reads_lt_1ms":4}
07:02:39.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.678999 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.710191 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.718156 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.039s user 0.038s sys 0.000s Metrics: {"bytes_written":39881,"cfile_init":1,"dirs.queue_time_us":60,"dirs.run_cpu_time_us":202,"dirs.run_wall_time_us":1718,"drs_written":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2186,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43,"spinlock_wait_cycles":325760}
07:02:39.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.719113 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 615 bytes on disk
07:02:39.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.719954 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":88,"lbm_reads_lt_1ms":4}
07:02:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.758620 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.048s user 0.043s sys 0.000s Metrics: {"bytes_written":39922,"cfile_init":1,"dirs.queue_time_us":48,"dirs.run_cpu_time_us":223,"dirs.run_wall_time_us":1537,"drs_written":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2586,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":46}
07:02:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.759575 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 650 bytes on disk
07:02:39.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.760227 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
07:02:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.846113 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:39.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.895368 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.049s user 0.046s sys 0.000s Metrics: {"bytes_written":39886,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1487,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2196,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:39.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.896364 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 595 bytes on disk
07:02:39.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.897047 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
07:02:39.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:39.971011 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.011253 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.026122 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.055s user 0.046s sys 0.004s Metrics: {"bytes_written":39752,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":254,"dirs.run_wall_time_us":1551,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2202,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.027515 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.028245 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4}
07:02:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.064146 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.052s user 0.049s sys 0.000s Metrics: {"bytes_written":39939,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":219,"dirs.run_wall_time_us":1657,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2310,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.064754 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 596 bytes on disk
07:02:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.065346 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4}
07:02:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.148331 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.187990 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.039s user 0.029s sys 0.007s Metrics: {"bytes_written":39962,"cfile_init":1,"dirs.queue_time_us":40,"dirs.run_cpu_time_us":206,"dirs.run_wall_time_us":1521,"drs_written":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1872,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41}
07:02:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.188673 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 596 bytes on disk
07:02:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.189314 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
07:02:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.279304 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.316239 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.320343 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.041s user 0.031s sys 0.007s Metrics: {"bytes_written":39790,"cfile_init":1,"dirs.queue_time_us":62,"dirs.run_cpu_time_us":211,"dirs.run_wall_time_us":1461,"drs_written":1,"lbm_read_time_us":93,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2336,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.321087 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.321789 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":71,"lbm_reads_lt_1ms":4}
07:02:40.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.363291 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.036s sys 0.003s Metrics: {"bytes_written":39784,"cfile_init":1,"dirs.queue_time_us":55,"dirs.run_cpu_time_us":208,"dirs.run_wall_time_us":1506,"drs_written":1,"lbm_read_time_us":82,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2876,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.363910 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.364573 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":97,"lbm_reads_lt_1ms":4}
07:02:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.440320 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.482113 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.042s user 0.040s sys 0.000s Metrics: {"bytes_written":39811,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":168,"dirs.run_wall_time_us":1394,"drs_written":1,"lbm_read_time_us":71,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2044,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:02:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.483286 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 572 bytes on disk
07:02:40.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.486549 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4}
07:02:40.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.572821 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.615501 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.621309 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.048s user 0.041s sys 0.004s Metrics: {"bytes_written":39817,"cfile_init":1,"dirs.queue_time_us":52,"dirs.run_cpu_time_us":203,"dirs.run_wall_time_us":1748,"drs_written":1,"lbm_read_time_us":86,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2320,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:40.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.622143 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 562 bytes on disk
07:02:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.623544 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4}
07:02:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.658502 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.043s user 0.039s sys 0.000s Metrics: {"bytes_written":39835,"cfile_init":1,"dirs.queue_time_us":51,"dirs.run_cpu_time_us":211,"dirs.run_wall_time_us":1432,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2826,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:02:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.660912 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 571 bytes on disk
07:02:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.661684 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
07:02:40.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.737699 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.799723 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.062s user 0.037s sys 0.011s Metrics: {"bytes_written":39844,"cfile_init":1,"dirs.queue_time_us":63,"dirs.run_cpu_time_us":207,"dirs.run_wall_time_us":1481,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2150,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.800724 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:40.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.801517 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
07:02:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.874759 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.912765 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.920599 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.035s sys 0.003s Metrics: {"bytes_written":39912,"cfile_init":1,"dirs.queue_time_us":46,"dirs.run_cpu_time_us":167,"dirs.run_wall_time_us":1478,"drs_written":1,"lbm_read_time_us":45,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1871,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:02:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.921273 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 606 bytes on disk
07:02:40.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.921872 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
07:02:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.958978 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.042s sys 0.000s Metrics: {"bytes_written":39914,"cfile_init":1,"dirs.queue_time_us":48,"dirs.run_cpu_time_us":153,"dirs.run_wall_time_us":1555,"drs_written":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2381,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":41,"spinlock_wait_cycles":896}
07:02:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.959683 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 596 bytes on disk
07:02:40.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:40.960814 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":82,"lbm_reads_lt_1ms":4}
07:02:41.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.052685 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.098420 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.045s user 0.044s sys 0.000s Metrics: {"bytes_written":39928,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":209,"dirs.run_wall_time_us":1595,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2057,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:02:41.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.099191 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 606 bytes on disk
07:02:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.100189 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4}
07:02:41.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.172894 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.211898 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.213258 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.040s user 0.037s sys 0.000s Metrics: {"bytes_written":39753,"cfile_init":1,"dirs.queue_time_us":54,"dirs.run_cpu_time_us":166,"dirs.run_wall_time_us":1815,"drs_written":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1941,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.214046 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.214732 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4}
07:02:41.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.252876 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.041s user 0.040s sys 0.000s Metrics: {"bytes_written":39808,"cfile_init":1,"dirs.queue_time_us":53,"dirs.run_cpu_time_us":205,"dirs.run_wall_time_us":1634,"drs_written":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2091,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.253721 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:41.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.254540 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
07:02:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.351418 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.396533 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.045s user 0.035s sys 0.006s Metrics: {"bytes_written":39733,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":1416,"drs_written":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1848,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.397369 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:41.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.398061 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
07:02:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.465876 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.505661 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.517293 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.051s user 0.038s sys 0.008s Metrics: {"bytes_written":39751,"cfile_init":1,"dirs.queue_time_us":49,"dirs.run_cpu_time_us":184,"dirs.run_wall_time_us":1601,"drs_written":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2235,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.518093 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:41.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.518864 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":84,"lbm_reads_lt_1ms":4}
07:02:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.559674 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.054s user 0.039s sys 0.007s Metrics: {"bytes_written":39751,"cfile_init":1,"dirs.queue_time_us":45,"dirs.run_cpu_time_us":180,"dirs.run_wall_time_us":1317,"drs_written":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2260,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.560421 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 582 bytes on disk
07:02:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.561254 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":81,"lbm_reads_lt_1ms":4}
07:02:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.649091 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.696244 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.047s user 0.044s sys 0.000s Metrics: {"bytes_written":39776,"cfile_init":1,"dirs.queue_time_us":62,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":1553,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2286,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:41.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.696980 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:41.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.697588 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4}
07:02:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.769976 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.812172 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.817538 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.047s user 0.041s sys 0.000s Metrics: {"bytes_written":39906,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1609,"drs_written":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2200,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:02:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.818595 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 606 bytes on disk
07:02:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.819710 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4}
07:02:41.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.863492 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.051s user 0.041s sys 0.000s Metrics: {"bytes_written":39928,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":170,"dirs.run_wall_time_us":1584,"drs_written":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1968,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:02:41.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.864281 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 606 bytes on disk
07:02:41.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.865103 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":83,"lbm_reads_lt_1ms":4}
07:02:41.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.949029 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.995430 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.041s sys 0.000s Metrics: {"bytes_written":39944,"cfile_init":1,"dirs.queue_time_us":344,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1580,"drs_written":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1825,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":42}
07:02:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.996570 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 606 bytes on disk
07:02:41.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:41.997299 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
07:02:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.071305 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.116822 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.127725 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.056s user 0.042s sys 0.004s Metrics: {"bytes_written":39852,"cfile_init":1,"dirs.queue_time_us":45,"dirs.run_cpu_time_us":228,"dirs.run_wall_time_us":1581,"drs_written":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2414,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.128912 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 560 bytes on disk
07:02:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.130188 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4}
07:02:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.163023 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.044s sys 0.000s Metrics: {"bytes_written":39895,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1554,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2210,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:42.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.163820 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 560 bytes on disk
07:02:42.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.164639 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":88,"lbm_reads_lt_1ms":4}
07:02:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.248432 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.293231 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.045s user 0.040s sys 0.000s Metrics: {"bytes_written":39792,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":199,"dirs.run_wall_time_us":1699,"drs_written":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1770,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":40}
07:02:42.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.293994 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 581 bytes on disk
07:02:42.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.294740 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4}
07:02:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.381528 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.415709 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.426256 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.044s user 0.040s sys 0.003s Metrics: {"bytes_written":39882,"cfile_init":1,"dirs.queue_time_us":61,"dirs.run_cpu_time_us":179,"dirs.run_wall_time_us":1509,"drs_written":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2042,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":44}
07:02:42.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.427322 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 626 bytes on disk
07:02:42.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.428174 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4}
07:02:42.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.462925 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.047s user 0.043s sys 0.000s Metrics: {"bytes_written":39964,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":193,"dirs.run_wall_time_us":1564,"drs_written":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2079,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":43}
07:02:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.464303 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 617 bytes on disk
07:02:42.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.465021 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4}
07:02:42.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.547423 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.595163 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.047s user 0.042s sys 0.004s Metrics: {"bytes_written":39838,"cfile_init":1,"dirs.queue_time_us":56,"dirs.run_cpu_time_us":237,"dirs.run_wall_time_us":1497,"drs_written":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2829,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39}
07:02:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.596004 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 572 bytes on disk
07:02:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.596912 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":73,"lbm_reads_lt_1ms":4}
07:02:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.679596 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.716251 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.770444 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.091s user 0.051s sys 0.001s Metrics: {"bytes_written":39826,"cfile_init":1,"dirs.queue_time_us":57,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1535,"drs_written":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7583,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":36}
07:02:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.771764 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 537 bytes on disk
07:02:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.778473 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4}
07:02:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.786700 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.069s user 0.042s sys 0.000s Metrics: {"bytes_written":39820,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":176,"dirs.run_wall_time_us":1343,"drs_written":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4606,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":35}
07:02:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.787509 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 526 bytes on disk
07:02:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.788709 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":99,"lbm_reads_lt_1ms":4}
07:02:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.848047 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:42.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.891438 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.043s user 0.031s sys 0.004s Metrics: {"bytes_written":39665,"cfile_init":1,"dirs.queue_time_us":58,"dirs.run_cpu_time_us":172,"dirs.run_wall_time_us":1494,"drs_written":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2112,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32}
07:02:42.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.892151 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 491 bytes on disk
07:02:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:42.892997 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":78,"lbm_reads_lt_1ms":4}
07:02:43.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.030000 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:43.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.039872 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.076485 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.046s user 0.042s sys 0.000s Metrics: {"bytes_written":39855,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":163,"dirs.run_wall_time_us":1483,"drs_written":1,"lbm_read_time_us":89,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2203,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":38}
07:02:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.077299 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 561 bytes on disk
07:02:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.078116 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":76,"lbm_reads_lt_1ms":4}
07:02:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.095614 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.055s user 0.040s sys 0.003s Metrics: {"bytes_written":39835,"cfile_init":1,"dirs.queue_time_us":45,"dirs.run_cpu_time_us":160,"dirs.run_wall_time_us":1904,"drs_written":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2293,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36}
07:02:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.096463 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 536 bytes on disk
07:02:43.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.097904 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":71,"lbm_reads_lt_1ms":4}
07:02:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.144150 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=1.000000
07:02:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.192447 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.048s user 0.031s sys 0.004s Metrics: {"bytes_written":39793,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":213,"dirs.run_wall_time_us":1440,"drs_written":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2149,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33}
07:02:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.193944 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 505 bytes on disk
07:02:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:43.194972 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
07:02:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.080905 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=0.969180
07:02:44.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.100935 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=0.969091
07:02:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.108034 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.027s user 0.025s sys 0.000s Metrics: {"bytes_written":39594,"cfile_init":1,"dirs.queue_time_us":363,"dirs.run_cpu_time_us":230,"dirs.run_wall_time_us":1475,"drs_written":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1655,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":26,"thread_start_us":254,"threads_started":1}
07:02:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.108911 9443 maintenance_manager.cc:419] P ccaf3b1f4b9e4673b9b942ff3becee0a: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 426 bytes on disk
07:02:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.109575 9332 maintenance_manager.cc:643] P ccaf3b1f4b9e4673b9b942ff3becee0a: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
07:02:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.159474 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.058s user 0.052s sys 0.004s Metrics: {"bytes_written":39582,"cfile_init":1,"dirs.queue_time_us":417,"dirs.run_cpu_time_us":163,"dirs.run_wall_time_us":1442,"drs_written":1,"lbm_read_time_us":92,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2715,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":25,"thread_start_us":296,"threads_started":1}
07:02:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.160426 9715 maintenance_manager.cc:419] P 036f0b11d4394c499f14ad934647290b: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 416 bytes on disk
07:02:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.161521 9603 maintenance_manager.cc:643] P 036f0b11d4394c499f14ad934647290b: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":105,"lbm_reads_lt_1ms":4}
07:02:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.198277 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff): perf score=0.969180
07:02:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.246836 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: FlushMRSOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.048s user 0.039s sys 0.008s Metrics: {"bytes_written":39259,"cfile_init":1,"dirs.queue_time_us":381,"dirs.run_cpu_time_us":187,"dirs.run_wall_time_us":1418,"drs_written":1,"lbm_read_time_us":92,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2707,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":14,"thread_start_us":261,"threads_started":1}
07:02:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.247833 9580 maintenance_manager.cc:419] P 918ddcbd9d30478f8c35fedac4798b89: Scheduling UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff): 298 bytes on disk
07:02:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:02:44.248999 9469 maintenance_manager.cc:643] P 918ddcbd9d30478f8c35fedac4798b89: UndoDeltaBlockGCOp(5495564ce7a540c3b9ebc2340735f3ff) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":112,"lbm_reads_lt_1ms":4}
07:02:48.249 [INFO - main] (TestKuduBackup.scala:963) nowMs: 1714114968249
07:02:48.250 [INFO - main] (TestKuduBackup.scala:965) propagated physicalMicros: 1714114963245222
07:02:48.250 [INFO - main] (TestKuduBackup.scala:966) propagated logical: 0
07:02:48.250 [INFO - main] (KuduBackup.scala:111) Backing up to root path: file:///tmp/backup7385703363266373782/
07:02:48.255 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 645a3ddf59634af1b28990c33346a513
07:02:48.255 [INFO - ForkJoinPool-243-worker-1] (KuduBackup.scala:55) Backing up table split-size-table to path: file:/tmp/backup7385703363266373782/645a3ddf59634af1b28990c33346a513-split-size-table/1714114968249
07:02:48.255 [INFO - ForkJoinPool-243-worker-1] (KuduBackup.scala:66) Looking for a previous backup: forceFull and fromMs options are not set.
07:02:48.255 [INFO - ForkJoinPool-243-worker-1] (KuduBackup.scala:73) No previous backup was found. Starting a full backup.
07:02:48.256 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-taskBMRGjA/java/build/dist-test/spark-warehouse').
07:02:48.256 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-taskBMRGjA/java/build/dist-test/spark-warehouse'.
07:02:48.276 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.277 [INFO - ForkJoinPool-243-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.277 [INFO - ForkJoinPool-243-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.278 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.278 [INFO - ForkJoinPool-243-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.278 [INFO - ForkJoinPool-243-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.278 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for split-size-table
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, 0x8000001E), 35260, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000001E, 0x80000043), 35551, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000043, 0x80000064), 35532, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000064, 0x80000091), 35732, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000091, 0x800000B7), 35596, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.305 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000B7, 0x800000DC), 35629, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000DC, 0x800000FF), 35574, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000FF, 0x80000125), 35647, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000125, 0x80000148), 35659, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000148, 0x8000016F), 35583, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000016F, 0x80000191), 35630, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000191, 0x800001B6), 35668, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001B6, 0x800001DE), 35580, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001DE, 0x80000204), 35637, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000204, 0x8000022D), 35706, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000022D, 0x80000256), 35782, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.306 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000256, 0x8000027D), 35631, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000027D, 0x800002A5), 35664, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002A5, 0x800002CF), 35748, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002CF, 0x800002F7), 35553, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002F7, 0x8000031F), 35596, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000031F, 0x80000349), 35764, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000349, 0x80000371), 35612, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000371, 0x80000398), 35658, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000398, 0x800003B8), 35485, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800003B8, 0x800003D9), 35613, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.307 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800003D9, <end>), 35079, "5495564ce7a540c3b9ebc2340735f3ff" [<start>, <end>)
07:02:48.324 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Starting job: save at KuduBackup.scala:97
07:02:48.325 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (save at KuduBackup.scala:97) with 27 output partitions
07:02:48.325 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (save at KuduBackup.scala:97)
07:02:48.325 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
07:02:48.325 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
07:02:48.325 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97), which has no missing parents
07:02:48.342 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 189.4 KiB, free 1993.7 MiB)
07:02:48.344 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 66.7 KiB, free 1993.6 MiB)
07:02:48.345 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-b8wt.c.gcp-upstream.internal:44167 (size: 66.7 KiB, free: 1993.8 MiB)
07:02:48.345 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
07:02:48.345 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 27 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))
07:02:48.346 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 27 tasks resource profile 0
07:02:48.347 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-b8wt.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5279 bytes) taskResourceAssignments Map()
07:02:48.348 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
07:02:48.359 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:02:48.361 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 645a3ddf59634af1b28990c33346a513
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
07:02:48.362 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:376) Parquet block size to 134217728
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:377) Parquet page size to 1048576
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:378) Parquet dictionary page size to 1048576
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:379) Dictionary is on
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:380) Validation is off
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:381) Writer version is: PARQUET_1_0
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:382) Maximum row group padding size is 8388608 bytes
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:383) Page size checking is: estimated
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:384) Min row count for page size check is: 100
07:02:48.363 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:385) Max row count for page size check is: 10000
07:02:48.364 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Initialized Parquet WriteSupport with Catalyst schema:
{
"type" : "struct",
"fields" : [ {
"name" : "key",
"type" : "integer",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c1_i",
"type" : "integer",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c2_s",
"type" : "string",
"nullable" : true,
"metadata" : { }
}, {
"name" : "c3_double",
"type" : "double",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c4_long",
"type" : "long",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c5_bool",
"type" : "boolean",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c6_short",
"type" : "short",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c7_float",
"type" : "float",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c8_binary",
"type" : "binary",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c9_unixtime_micros",
"type" : "timestamp",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c10_byte",
"type" : "byte",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c11_decimal32",
"type" : "decimal(9,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c12_decimal64",
"type" : "decimal(18,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c13_decimal128",
"type" : "decimal(38,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c14_varchar",
"type" : "string",
"nullable" : true,
"metadata" : { }
}, {
"name" : "c15_date",
"type" : "date",
"nullable" : false,
"metadata" : { }
} ]
}
and corresponding Parquet message type:
message spark_schema {
required int32 key;
required int32 c1_i;
optional binary c2_s (UTF8);
required double c3_double;
required int64 c4_long;
required boolean c5_bool;
required int32 c6_short (INT_16);
required float c7_float;
required binary c8_binary;
required int96 c9_unixtime_micros;
required int32 c10_byte (INT_8);
required int32 c11_decimal32 (DECIMAL(9,0));
required int64 c12_decimal64 (DECIMAL(18,0));
required fixed_len_byte_array(16) c13_decimal128 (DECIMAL(38,0));
optional binary c14_varchar (UTF8);
required int32 c15_date (DATE);
}
07:02:48.374 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:48.378 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] Successfully connected to peer
07:02:48.387 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:48.389 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6c206e0a, L:/127.0.0.1:48572 - R:/127.8.209.3:43753] using TOKEN/null
07:02:48.419 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false, responseScanTimestamp = 7021014909947904000) for scanner KuduScanner(table=split-size-table, tablet=5495564ce7a540c3b9ebc2340735f3ff, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x8000001E, lastPrimaryKey=<last>)
07:02:48.441 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (InternalParquetRecordWriter.java:165) Flushing mem columnStore to file. allocated memory: 38711
07:02:48.444 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:593) Saved output of task 'attempt_202404260702481392718883077737376_0000_m_000000_0' to file:/tmp/backup7385703363266373782/645a3ddf59634af1b28990c33346a513-split-size-table/1714114968249/_temporary/0/task_202404260702481392718883077737376_0000_m_000000
07:02:48.444 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) attempt_202404260702481392718883077737376_0000_m_000000_0: Committed
07:02:48.445 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 6297 bytes result sent to driver
07:02:48.446 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5285 bytes) taskResourceAssignments Map()
07:02:48.446 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
07:02:48.446 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 99 ms on dist-test-slave-b8wt.c.gcp-upstream.internal (executor driver) (1/27)
07:02:48.454 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:02:48.456 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 645a3ddf59634af1b28990c33346a513
07:02:48.472 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.472 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.473 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.473 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:02:48.473 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:02:48.473 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:02:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46: runtime error: member call on null pointer of type 'kudu::EncodedKey'
07:02:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7efbfca07d53 in kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46
07:02:49.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x7efc07719048 in kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11
07:02:49.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7efc0771809f in kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3
07:02:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7efc0796122d in kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3
07:02:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7efbfc8e5fe3 in kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3
07:02:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7efbfc8dbbdc in kudu::InitAndMaybeWrap(std::unique_ptr<kudu::RowwiseIterator, std::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3
07:02:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7efbfc8e2f41 in kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5
07:02:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7efbfc8e1f0d in kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3
07:02:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7efc0757000d in kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3
07:02:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7efc0a06120d in kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15
07:02:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7efc0a05b40b in kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16
07:02:49.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7efc047f3556 in kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/asan/src/kudu/tserver/tserver_service.service.cc:263:13
07:02:49.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7efc047f3285 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7efc0182f4fe in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x7efc0182e3de in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3
07:02:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7efc018348b5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15
07:02:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7efc018351df in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5
07:02:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7efc01835008 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 0xa6022c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 0x7efbfab50c1d in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3
07:02:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 0x7efbfded36da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
07:02:49.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 0x7efbf6d1571e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
07:02:49.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:49.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46 in
07:02:49.680 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] handling channelInactive
07:02:49.680 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state READY due to: connection closed
07:02:49.681 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: connection closed
07:02:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46: runtime error: member call on null pointer of type 'kudu::EncodedKey'
07:02:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:49.800302 9468 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.209.3:43753: connect: Connection refused (error 111)
07:02:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:02:49.804848 9468 consensus_peers.cc:485] T 5495564ce7a540c3b9ebc2340735f3ff P 918ddcbd9d30478f8c35fedac4798b89 -> Peer 036f0b11d4394c499f14ad934647290b (127.8.209.3:43753): Couldn't send request to peer 036f0b11d4394c499f14ad934647290b. Status: Network error: Client connection negotiation failed: client connection to 127.8.209.3:43753: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:02:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7ffb0a425d53 in kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46
07:02:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x7ffb15137048 in kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11
07:02:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7ffb1513609f in kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3
07:02:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7ffb1537f22d in kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3
07:02:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7ffb0a303fe3 in kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3
07:02:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7ffb0a2f9bdc in kudu::InitAndMaybeWrap(std::unique_ptr<kudu::RowwiseIterator, std::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3
07:02:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7ffb0a300f41 in kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5
07:02:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7ffb0a2fff0d in kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3
07:02:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7ffb14f8e00d in kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3
07:02:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7ffb17a7f20d in kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15
07:02:50.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7ffb17a7940b in kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16
07:02:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7ffb12211556 in kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/asan/src/kudu/tserver/tserver_service.service.cc:263:13
07:02:50.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7ffb12211285 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7ffb0f24d4fe in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x7ffb0f24c3de in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3
07:02:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7ffb0f2528b5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15
07:02:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7ffb0f2531df in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5
07:02:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7ffb0f253008 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 0xa6022c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 0x7ffb0856ec1d in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3
07:02:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 0x7ffb0b8f16da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
07:02:50.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 0x7ffb0473371e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
07:02:50.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:50.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46 in
07:02:50.914 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] handling channelInactive
07:02:50.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state READY due to: connection closed
07:02:50.914 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: connection closed
07:02:50.923 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:02:50.924 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] Successfully connected to peer
07:02:50.936 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:02:50.938 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf3dfe8e5, L:/127.0.0.1:34440 - R:/127.8.209.1:41725] using TOKEN/null
07:02:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46: runtime error: member call on null pointer of type 'kudu::EncodedKey'
07:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f91f81aed53 in kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46
07:02:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x7f9202ec0048 in kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11
07:02:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f9202ebf09f in kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3
07:02:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f920310822d in kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3
07:02:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f91f808cfe3 in kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3
07:02:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f91f8082bdc in kudu::InitAndMaybeWrap(std::unique_ptr<kudu::RowwiseIterator, std::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3
07:02:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f91f8089f41 in kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5
07:02:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f91f8088f0d in kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3
07:02:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f9202d1700d in kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3
07:02:51.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f920580820d in kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15
07:02:51.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f920580240b in kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16
07:02:51.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f91fff9a556 in kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/asan/src/kudu/tserver/tserver_service.service.cc:263:13
07:02:51.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f91fff9a285 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f91fcfd64fe in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x7f91fcfd53de in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3
07:02:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f91fcfdb8b5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15
07:02:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f91fcfdc1df in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5
07:02:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f91fcfdc008 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:02:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 0xa6022c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:02:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 0x7f91f62f7c1d in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3
07:02:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 0x7f91f967a6da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
07:02:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 0x7f91f24bc71e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
07:02:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:02:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:46 in
07:02:52.133 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] handling channelInactive
07:02:52.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state READY due to: connection closed
07:02:52.133 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: connection closed
07:02:52.144 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:52.145 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:52.145 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=4, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=4, TimeoutTracker(timeout=30000, elapsed=3672), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1)))) after lookup
07:02:52.146 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:52.147 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.148 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.163 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:02:52.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.164 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.165 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.222 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:02:52.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:52.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:52.224 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:52.245 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:52.245 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:52.245 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=8, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=8, TimeoutTracker(timeout=30000, elapsed=3772), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868)))) after lookup
07:02:52.246 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:52.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.246 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:52.743 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:02:52.744 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.744 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.744 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:52.862 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:02:52.864 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:52.864 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:52.864 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:54.364 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:54.364 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:54.365 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=12, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=12, TimeoutTracker(timeout=30000, elapsed=5892), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646)))) after lookup
07:02:54.366 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:54.367 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:54.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:54.367 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:54.543 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:02:54.544 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:54.544 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:54.544 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:55.443 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:02:55.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:55.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:55.444 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:55.565 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:55.565 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:55.565 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=16, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=16, TimeoutTracker(timeout=30000, elapsed=7092), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992)))) after lookup
07:02:55.566 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:55.567 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:55.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:55.567 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:56.123 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:02:56.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:56.124 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:56.124 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:02:59.463 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:02:59.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:59.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:59.464 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:02:59.904 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:02:59.904 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:02:59.904 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=20, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=20, TimeoutTracker(timeout=30000, elapsed=11432), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275)))) after lookup
07:02:59.906 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:02:59.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:59.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:02:59.907 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:00.443 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:03:00.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:00.444 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:00.444 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:04.023 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:03:04.024 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:04.024 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:04.024 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:04.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:03:04.325 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:03:04.325 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=24, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=24, TimeoutTracker(timeout=30000, elapsed=15852), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347)))) after lookup
07:03:04.326 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:03:04.327 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:04.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:04.327 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:08.303 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:03:08.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:08.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:08.304 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:09.363 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:03:09.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:09.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:09.364 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:12.624 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:03:12.624 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:03:12.625 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=28, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=28, TimeoutTracker(timeout=30000, elapsed=24152), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612)))) after lookup
07:03:12.626 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:03:12.627 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:12.627 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:12.627 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:14.902 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:03:14.903 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:14.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:14.904 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:15.183 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] connecting to peer
07:03:15.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:15.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] cleaning up while in state CONNECTING due to: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:15.184 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
07:03:15.185 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
07:03:15.186 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:03:15.191 [ERROR - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:94) Exception in task 1.0 in stage 0.0 (TID 1)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144) ~[main/:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:03:15.239 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-b8wt.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5285 bytes) taskResourceAssignments Map()
07:03:15.239 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
07:03:15.242 [WARN - task-result-getter-1] (Logging.scala:69) Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:03:15.246 [ERROR - task-result-getter-1] (Logging.scala:73) Task 1 in stage 0.0 failed 1 times; aborting job
07:03:15.249 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
07:03:15.249 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
07:03:15.252 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:03:15.253 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Executor is trying to kill task 2.0 in stage 0.0 (TID 2), reason: Stage cancelled
07:03:15.254 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
07:03:15.254 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 645a3ddf59634af1b28990c33346a513
07:03:15.254 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (save at KuduBackup.scala:97) failed in 26.928 s due to Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
07:03:15.256 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:03:15.256 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:03:15.256 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:03:15.256 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:03:15.256 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:03:15.257 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:03:15.257 [INFO - ForkJoinPool-243-worker-1] (Logging.scala:57) Job 0 failed: save at KuduBackup.scala:97, took 26.932364 s
07:03:15.258 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 5495564ce7a540c3b9ebc2340735f3ff for table 'split-size-table' with partition [<start>, <end>)
07:03:15.258 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5495564ce7a540c3b9ebc2340735f3ff@[036f0b11d4394c499f14ad934647290b(127.8.209.3:43753),918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)[L],ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)]}]
07:03:15.258 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [1ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, deferred=null)) after lookup
07:03:15.259 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] connecting to peer
07:03:15.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:15.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] cleaning up while in state CONNECTING due to: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
07:03:15.259 [ERROR - ForkJoinPool-243-worker-1] (Logging.scala:94) Aborting job 9f93b5a7-daa9-4706-acc8-989e95b11b30.
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
07:03:15.260 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
at scala.Option.foreach(Option.scala:407)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
07:03:15.265 [ERROR - ForkJoinPool-243-worker-1] (KuduBackup.scala:154) Failed to back up table split-size-table
org.apache.spark.SparkException: Job aborted.
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
... 64 more
org.apache.spark.SparkException: Job aborted.
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-b8wt.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, KuduRpc(method=Scan, tablet=5495564ce7a540c3b9ebc2340735f3ff, attempt=31, TimeoutTracker(timeout=30000, elapsed=26711), Traces: [0ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [1207ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: connection closed, [1208ms] delaying RPC due to: Network error: connection closed, [1209ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [2441ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: connection closed, [2441ms] delaying RPC due to: Network error: connection closed, [2450ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3660ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: connection closed, [3660ms] delaying RPC due to: Network error: connection closed, [3670ms] refreshing cache from master, [3670ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3671ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3673ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3675ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3675ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3690ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [3691ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3692ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [3749ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [3751ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3751ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [3770ms] refreshing cache from master, [3770ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [3771ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [3772ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [3773ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [3774ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [4270ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [4271ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4271ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [4389ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [4391ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [4391ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [5889ms] refreshing cache from master, [5889ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [5891ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [5893ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [5894ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [5894ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [6070ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [6071ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6071ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [6969ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [6971ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [6971ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [7090ms] refreshing cache from master, [7090ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [7092ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [7093ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [7094ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7094ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [7650ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [7651ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [7651ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [10990ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [10991ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [10991ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [11429ms] refreshing cache from master, [11429ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [11431ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [11432ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [11434ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11434ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [11969ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [11971ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [11971ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [15550ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [15551ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15551ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [15850ms] refreshing cache from master, [15850ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [15851ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [15853ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [15854ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [15854ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [19830ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [19831ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [19831ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [20890ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [20891ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [20891ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [24149ms] refreshing cache from master, [24149ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.209.60:37981, [24151ms] Sub RPC GetTableLocations: received response from server master-127.8.209.60:37981: OK, [24153ms] sending RPC to server 036f0b11d4394c499f14ad934647290b, [24154ms] received response from server 036f0b11d4394c499f14ad934647290b: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [24154ms] delaying RPC due to: Network error: Failed to connect to peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753): Connection refused: /127.8.209.3:43753, [26429ms] sending RPC to server 918ddcbd9d30478f8c35fedac4798b89, [26431ms] received response from server 918ddcbd9d30478f8c35fedac4798b89: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26431ms] delaying RPC due to: Network error: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883, [26710ms] sending RPC to server ccaf3b1f4b9e4673b9b942ff3becee0a, [26711ms] received response from server ccaf3b1f4b9e4673b9b942ff3becee0a: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, [26711ms] delaying RPC due to: Network error: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725, deferred=Deferred@262592774(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@578354268 after retry RPC@1113812868) -> (continuation of Deferred@2145661349 after retry RPC@885313646) -> (continuation of Deferred@297798466 after retry RPC@1122381992) -> (continuation of Deferred@1868360562 after retry RPC@694216275) -> (continuation of Deferred@1571878099 after retry RPC@1150266347) -> (continuation of Deferred@317354660 after retry RPC@1930950612) -> (continuation of Deferred@801848558 after retry RPC@1741196412))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725): Connection refused: /127.8.209.1:41725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.209.1:41725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
at scala.Option.foreach(Option.scala:407)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
... 64 more
07:03:15.270 [ERROR - ForkJoinPool-243-worker-1] (KuduBackup.scala:162) Failed to back up table split-size-table: Look back in the logs for the full exception. Error: Failure(org.apache.spark.SparkException: Job aborted.)
07:03:15.274 [INFO - dispatcher-event-loop-1] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
07:03:15.278 [INFO - main] (Logging.scala:57) MemoryStore cleared
07:03:15.278 [INFO - main] (Logging.scala:57) BlockManager stopped
07:03:15.279 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
07:03:15.279 [INFO - dispatcher-event-loop-0] (Logging.scala:57) OutputCommitCoordinator stopped!
07:03:15.283 [DEBUG - pool-1012-thread-1] (Connection.java:748) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] connecting to peer
07:03:15.283 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
07:03:15.283 [DEBUG - main] (Connection.java:575) [peer master-127.8.209.60:37981(127.8.209.60:37981)] disconnecting while in state READY
07:03:15.283 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:15.283 [DEBUG - main] (Connection.java:575) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] disconnecting while in state CONNECTING
07:03:15.284 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883)] cleaning up while in state CONNECTING due to: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:15.284 [DEBUG - main] (Connection.java:575) [peer 036f0b11d4394c499f14ad934647290b(127.8.209.3:43753)] disconnecting while in state TERMINATED
07:03:15.284 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883) for tablet 5495564ce7a540c3b9ebc2340735f3ff: Failed to connect to peer 918ddcbd9d30478f8c35fedac4798b89(127.8.209.2:39883): Connection refused: /127.8.209.2:39883
07:03:15.284 [DEBUG - main] (Connection.java:575) [peer ccaf3b1f4b9e4673b9b942ff3becee0a(127.8.209.1:41725)] disconnecting while in state TERMINATED
07:03:15.284 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.209.60:37981(127.8.209.60:37981)] handling channelInactive
07:03:15.284 [DEBUG - main] (Connection.java:575) [peer master-127.8.209.61:33187(127.8.209.61:33187)] disconnecting while in state READY
07:03:15.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.209.60:37981(127.8.209.60:37981)] cleaning up while in state READY due to: connection closed
07:03:15.284 [DEBUG - main] (Connection.java:575) [peer master-127.8.209.62:33719(127.8.209.62:33719)] disconnecting while in state READY
07:03:15.285 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:03:15.286 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.209.61:33187(127.8.209.61:33187)] handling channelInactive
07:03:15.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.209.61:33187(127.8.209.61:33187)] cleaning up while in state READY due to: connection closed
07:03:15.286 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.209.62:33719(127.8.209.62:33719)] handling channelInactive
07:03:15.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.209.62:33719(127.8.209.62:33719)] cleaning up while in state READY due to: connection closed
07:03:15.286 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 28 thread-local buffer(s) from thread: kudu-nio-0
07:03:15.286 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 34 thread-local buffer(s) from thread: kudu-nio-1
07:03:15.296 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:03:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.297482 9028 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu pid 9039
07:03:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.299248 9100 generic_service.cc:196] Checking for leaks (request via RPC)
07:03:15.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.416079 9028 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu with pid 9039
07:03:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.439025 9028 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu pid 9107
07:03:15.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.440162 9173 generic_service.cc:196] Checking for leaks (request via RPC)
07:03:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.570392 9028 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu with pid 9107
07:03:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.594138 9028 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu pid 9180
07:03:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.595208 9244 generic_service.cc:196] Checking for leaks (request via RPC)
07:03:15.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:03:15.713919 9028 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBMRGjA/build/asan/bin/kudu with pid 9180
07:03:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:03:15Z chronyd exiting
07:03:15.882 [ERROR - main] (RetryRule.java:206) org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes: failed attempt 1
java.lang.AssertionError: null
at org.junit.Assert.fail(Assert.java:87) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:53) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414) ~[test/:?]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.AssertionError
at org.junit.Assert.fail(Assert.java:87)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.junit.Assert.assertTrue(Assert.java:53)
at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)