07:09:02.576 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:09:02.576 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskRfRc_3/test-tmp
07:09:02.577 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:09:02.577 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskRfRc_3/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:09:02.579 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768"
extra_tserver_flags: "--flush_threshold_mb=1"
extra_tserver_flags: "--flush_threshold_secs=1"
extra_tserver_flags: "--enable_rowset_compaction=false"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:09:02.904 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:09:02.905 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:09:02.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:02.909755 3252 mini_oidc.cc:152] Starting JWKS server
07:09:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:02.913470 3252 webserver.cc:460] Webserver started at http://127.0.0.1:39181/ using document root <none> and password file <none>
07:09:02.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:02.913991 3252 mini_oidc.cc:189] Starting OIDC Discovery server
07:09:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:02.915894 3252 webserver.cc:460] Webserver started at http://127.0.0.1:32867/ using document root <none> and password file <none>
07:09:02.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:09:02Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:09:02.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:09:02Z Disabled control of system clock
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:02.982650 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/logs
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data/info.pb
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.62:33155
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.62
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json with env {}
07:09:03.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.288643 3264 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:03.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.289208 3264 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:03.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.289702 3264 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:03.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.323197 3264 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:09:03.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.323567 3264 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.323907 3264 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:09:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.324210 3264 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:09:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.324487 3264 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:09:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.324698 3264 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39181/jwks.json
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:03.359503 3264 master_runner.cc:386] Master server non-default flags:
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.62:33155
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data/info.pb
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.62
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/logs
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:03.361084 3264 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:03.362752 3264 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:03.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.378077 3271 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.378710 3270 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:03.379973 3264 server_base.cc:1034] running on GCE node
07:09:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:03.379288 3273 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.488806 3264 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:04.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.491971 3264 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.493443 3264 hybrid_clock.cc:648] HybridClock initialized: now 1714115344493385 us; error 78 us; skew 500 ppm
07:09:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.494314 3264 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.507126 3264 webserver.cc:460] Webserver started at http://127.3.45.62:45615/ using document root <none> and password file <none>
07:09:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.508133 3264 fs_manager.cc:359] Metadata directory not provided
07:09:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.508332 3264 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.508729 3264 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.513679 3264 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data/instance:
07:09:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6149d28a1f6a434ab0dd88551148c905"
07:09:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:04 on dist-test-slave-hqrx"
07:09:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.514822 3264 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal/instance:
07:09:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6149d28a1f6a434ab0dd88551148c905"
07:09:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:04 on dist-test-slave-hqrx"
07:09:04.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.522176 3264 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
07:09:04.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.528436 3280 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.529773 3264 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.530107 3264 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6149d28a1f6a434ab0dd88551148c905"
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:04 on dist-test-slave-hqrx"
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.530458 3264 fs_report.cc:383] FS layout report
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data/data
07:09:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.587280 3264 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.588801 3264 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.589217 3264 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.658215 3264 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.62:33155
07:09:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.658318 3331 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.62:33155 every 8 connection(s)
07:09:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.661033 3264 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/data/info.pb
07:09:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.666167 3332 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:04.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.667433 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3264
07:09:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.667917 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-0/wal/instance
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.671231 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/logs
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data/info.pb
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:04.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.61:42905
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.61
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json with env {}
07:09:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.678069 3332 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } has no permanent_uuid. Determining permanent_uuid...
07:09:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.703678 3332 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } has no permanent_uuid. Determining permanent_uuid...
07:09:04.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.706192 3281 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.710459 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.760061 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 1
07:09:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.764462 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.833001 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 2
07:09:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.837159 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:04.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:04.928661 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 3
07:09:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.932837 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.980341 3334 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.980907 3334 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:04.981403 3334 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.011508 3334 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:09:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.011801 3334 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.012058 3334 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:09:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.012265 3334 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:09:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.012465 3334 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:09:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.012609 3334 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39181/jwks.json
07:09:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.047400 3334 master_runner.cc:386] Master server non-default flags:
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.61:42905
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data/info.pb
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.61
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/logs
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.048712 3334 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.050369 3334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.066123 3343 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:05.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.076345 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 4
07:09:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.080140 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.379663 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 5
07:09:05.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.384624 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.932044 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 6
07:09:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.940318 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.61:42905: Network error: Client connection negotiation failed: client connection to 127.3.45.61:42905: connect: Connection refused (error 111)
07:09:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:05.068205 3334 server_base.cc:1034] running on GCE node
07:09:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.066937 3346 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:05.067767 3344 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.194306 3334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.197759 3334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.199249 3334 hybrid_clock.cc:648] HybridClock initialized: now 1714115346199173 us; error 88 us; skew 500 ppm
07:09:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.200158 3334 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.207032 3334 webserver.cc:460] Webserver started at http://127.3.45.61:32817/ using document root <none> and password file <none>
07:09:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.208020 3334 fs_manager.cc:359] Metadata directory not provided
07:09:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.208212 3334 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.208616 3334 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.213498 3334 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data/instance:
07:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ec4db5ea9874f88b2804e7240700d85"
07:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:06 on dist-test-slave-hqrx"
07:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.214552 3334 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal/instance:
07:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ec4db5ea9874f88b2804e7240700d85"
07:09:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:06 on dist-test-slave-hqrx"
07:09:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.221895 3334 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
07:09:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.227739 3355 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.228921 3334 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.229238 3334 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ec4db5ea9874f88b2804e7240700d85"
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:06 on dist-test-slave-hqrx"
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.229558 3334 fs_report.cc:383] FS layout report
07:09:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data/data
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.299170 3334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.300644 3334 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:06.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.301064 3334 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.371006 3334 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.61:42905
07:09:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.371090 3406 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.61:42905 every 8 connection(s)
07:09:06.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.374058 3334 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/data/info.pb
07:09:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.376942 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3334
07:09:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.377512 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-1/wal/instance
07:09:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.379606 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/logs
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data/info.pb
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.60:44303
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.60
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json with env {}
07:09:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.379974 3407 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.391539 3407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } has no permanent_uuid. Determining permanent_uuid...
07:09:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.413448 3407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } has no permanent_uuid. Determining permanent_uuid...
07:09:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.428015 3407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } has no permanent_uuid. Determining permanent_uuid...
07:09:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.430568 3358 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.434758 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:06.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.484261 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 1
07:09:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.488471 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.556867 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 2
07:09:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.561138 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.652566 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 3
07:09:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.657305 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.704542 3409 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.705103 3409 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.705719 3409 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.736021 3409 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:09:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.736326 3409 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.736524 3409 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:09:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.736725 3409 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:09:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.736923 3409 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:09:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.737069 3409 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39181/jwks.json
07:09:06.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.771562 3409 master_runner.cc:386] Master server non-default flags:
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.60:44303
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39181/jwks.json
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data/info.pb
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.60
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/logs
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.772850 3409 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.774505 3409 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.790401 3422 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.800711 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 4
07:09:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.804805 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.001055 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } attempt: 7
07:09:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.014323 3332 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } has no permanent_uuid. Determining permanent_uuid...
07:09:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.021149 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.079593 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 1
07:09:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.084486 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.104287 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 5
07:09:07.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.109781 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.165892 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 2
07:09:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.170076 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.255759 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 3
07:09:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.260663 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.401314 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 4
07:09:07.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.406383 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.657184 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 6
07:09:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.664117 3407 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.690071 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 5
07:09:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:07.694850 3332 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.3.45.60:44303: Network error: Client connection negotiation failed: client connection to 127.3.45.60:44303: connect: Connection refused (error 111)
07:09:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:06.791726 3409 server_base.cc:1034] running on GCE node
07:09:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.791220 3419 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:06.790529 3420 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.916966 3409 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.919720 3409 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.921145 3409 hybrid_clock.cc:648] HybridClock initialized: now 1714115347921114 us; error 37 us; skew 500 ppm
07:09:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.922012 3409 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:07.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.928400 3409 webserver.cc:460] Webserver started at http://127.3.45.60:35079/ using document root <none> and password file <none>
07:09:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.929397 3409 fs_manager.cc:359] Metadata directory not provided
07:09:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.929637 3409 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.930128 3409 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.935163 3409 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data/instance:
07:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ad034f1c404d6f9cbc29366752b96a"
07:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:07 on dist-test-slave-hqrx"
07:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.936496 3409 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal/instance:
07:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ad034f1c404d6f9cbc29366752b96a"
07:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:07 on dist-test-slave-hqrx"
07:09:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.943831 3409 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
07:09:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.949476 3434 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.950572 3409 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.950924 3409 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13ad034f1c404d6f9cbc29366752b96a"
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:07 on dist-test-slave-hqrx"
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:07.951272 3409 fs_report.cc:383] FS layout report
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal
07:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data/data
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.014606 3409 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.016372 3409 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:08.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.016897 3409 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.087107 3409 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.60:44303
07:09:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.087203 3485 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.60:44303 every 8 connection(s)
07:09:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.090042 3409 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/data/info.pb
07:09:08.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.094435 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3409
07:09:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.094854 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/master-2/wal/instance
07:09:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.095162 3486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:08.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.113662 3486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } has no permanent_uuid. Determining permanent_uuid...
07:09:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.138499 3486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } has no permanent_uuid. Determining permanent_uuid...
07:09:08.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.148262 3486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } has no permanent_uuid. Determining permanent_uuid...
07:09:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.165468 3486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: Bootstrap starting.
07:09:08.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.171644 3486 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: Neither blocks nor log segments found. Creating new log.
07:09:08.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.173722 3486 log.cc:824] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: Log is configured to *not* fsync() on all Append() calls
07:09:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.178323 3486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: No bootstrap required, opened a new log
07:09:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.200712 3486 raft_consensus.cc:348] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.201357 3486 raft_consensus.cc:374] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.201550 3486 raft_consensus.cc:729] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 13ad034f1c404d6f9cbc29366752b96a, State: Initialized, Role: FOLLOWER
07:09:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.202200 3486 consensus_queue.cc:260] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.204550 3495 sys_catalog.cc:455] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.205201 3495 sys_catalog.cc:458] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: This master's current role is: FOLLOWER
07:09:08.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.206329 3486 sys_catalog.cc:564] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: configured and running, proceeding with master startup.
07:09:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.231572 3506 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:09:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.231966 3506 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:09:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.247476 3332 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 6
07:09:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.226821 3409 master_runner.cc:186] Error getting master registration for 127.3.45.62:33155: OK, instance_id { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" instance_seqno: 1714115344638911 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:09:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.253827 3409 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:09:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.265744 3332 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: Bootstrap starting.
07:09:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.271473 3332 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: Neither blocks nor log segments found. Creating new log.
07:09:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.272179 3440 tablet.cc:2369] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:09:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.274184 3332 log.cc:824] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: Log is configured to *not* fsync() on all Append() calls
07:09:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.281219 3332 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: No bootstrap required, opened a new log
07:09:08.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.314100 3332 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.315039 3332 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:08.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.315369 3332 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6149d28a1f6a434ab0dd88551148c905, State: Initialized, Role: FOLLOWER
07:09:08.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.316344 3332 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.319249 3510 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.319927 3510 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: This master's current role is: FOLLOWER
07:09:08.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.321441 3332 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: configured and running, proceeding with master startup.
07:09:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.348693 3286 tablet.cc:2369] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:09:08.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.351176 3521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:09:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:08.351774 3521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:09:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.351094 3264 master_runner.cc:186] Error getting master registration for 127.3.45.61:42905: OK, instance_id { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" instance_seqno: 1714115346351488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:09:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.381742 3264 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:09:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.385885 3510 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:09:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.386499 3510 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.389168 3510 leader_election.cc:288] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9ec4db5ea9874f88b2804e7240700d85 (127.3.45.61:42905), 13ad034f1c404d6f9cbc29366752b96a (127.3.45.60:44303)
07:09:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.393584 3461 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6149d28a1f6a434ab0dd88551148c905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "13ad034f1c404d6f9cbc29366752b96a" is_pre_election: true
07:09:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.394654 3461 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6149d28a1f6a434ab0dd88551148c905 in term 0.
07:09:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.396530 3283 leader_election.cc:302] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 13ad034f1c404d6f9cbc29366752b96a, 6149d28a1f6a434ab0dd88551148c905; no voters:
07:09:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.397843 3510 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:09:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.398599 3510 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:09:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.399330 3510 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 0 FOLLOWER]: Advancing to term 1
07:09:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.412067 3510 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.415766 3461 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6149d28a1f6a434ab0dd88551148c905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "13ad034f1c404d6f9cbc29366752b96a"
07:09:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.416378 3461 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 0 FOLLOWER]: Advancing to term 1
07:09:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.428968 3461 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6149d28a1f6a434ab0dd88551148c905 in term 1.
07:09:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.437901 3510 leader_election.cc:288] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [CANDIDATE]: Term 1 election: Requested vote from peers 9ec4db5ea9874f88b2804e7240700d85 (127.3.45.61:42905), 13ad034f1c404d6f9cbc29366752b96a (127.3.45.60:44303)
07:09:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.438321 3283 leader_election.cc:302] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 13ad034f1c404d6f9cbc29366752b96a, 6149d28a1f6a434ab0dd88551148c905; no voters:
07:09:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.439147 3510 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 1 FOLLOWER]: Leader election won for term 1
07:09:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.442301 3510 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [term 1 LEADER]: Becoming Leader. State: Replica: 6149d28a1f6a434ab0dd88551148c905, State: Running, Role: LEADER
07:09:08.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.443492 3510 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.454037 3523 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6149d28a1f6a434ab0dd88551148c905. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.454677 3523 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: This master's current role is: LEADER
07:09:08.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.473639 3525 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:09:08.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.481889 3525 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:09:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.509552 3461 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [term 1 FOLLOWER]: Refusing update from remote peer 6149d28a1f6a434ab0dd88551148c905: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:09:08.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.511276 3523 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [LEADER]: Connected to new peer: Peer: permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:09:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.537230 3495 sys_catalog.cc:455] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6149d28a1f6a434ab0dd88551148c905. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.538055 3495 sys_catalog.cc:458] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: This master's current role is: FOLLOWER
07:09:08.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.546145 3526 mvcc.cc:205] Tried to move back new op lower bound from 7021016467477274624 to 7021016467235618816. Current Snapshot: MvccSnapshot[applied={T|T < 7021016467477274624}]
07:09:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.548167 3510 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.549182 3510 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: This master's current role is: LEADER
07:09:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.553925 3525 catalog_manager.cc:1347] Generated new cluster ID: 8840025a8e804e269c1ed90a4d837e4f
07:09:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.554230 3525 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:09:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.553655 3527 mvcc.cc:205] Tried to move back new op lower bound from 7021016467477274624 to 7021016467235618816. Current Snapshot: MvccSnapshot[applied={T|T < 7021016467477274624}]
07:09:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.576742 3495 sys_catalog.cc:455] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.577522 3495 sys_catalog.cc:458] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a [sys.catalog]: This master's current role is: FOLLOWER
07:09:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.590914 3525 catalog_manager.cc:1370] Generated new certificate authority record
07:09:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.593067 3525 catalog_manager.cc:1504] Loading token signing keys...
07:09:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.620167 3525 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905: Generated new TSK 0
07:09:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.621331 3525 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.648701 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/logs
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data/info.pb
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.1:0
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.1
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.1
07:09:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:09:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.724484 3407 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } attempt: 7
07:09:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.771353 3407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: Bootstrap starting.
07:09:08.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.782430 3407 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: Neither blocks nor log segments found. Creating new log.
07:09:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.790412 3407 log.cc:824] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: Log is configured to *not* fsync() on all Append() calls
07:09:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.809547 3407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: No bootstrap required, opened a new log
07:09:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.871914 3407 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.880329 3407 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.881934 3407 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ec4db5ea9874f88b2804e7240700d85, State: Initialized, Role: FOLLOWER
07:09:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.885723 3407 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } }
07:09:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.908902 3534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:08.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.909790 3534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: This master's current role is: FOLLOWER
07:09:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.917380 3380 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [term 0 FOLLOWER]: Advancing to term 1
07:09:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.920878 3382 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6149d28a1f6a434ab0dd88551148c905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ec4db5ea9874f88b2804e7240700d85" is_pre_election: true
07:09:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.922436 3381 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6149d28a1f6a434ab0dd88551148c905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ec4db5ea9874f88b2804e7240700d85"
07:09:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.929626 3380 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [term 1 FOLLOWER]: Refusing update from remote peer 6149d28a1f6a434ab0dd88551148c905: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:09:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.931731 3407 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: configured and running, proceeding with master startup.
07:09:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:08.931558 3523 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.041229 3534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6149d28a1f6a434ab0dd88551148c905. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.041327 3540 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.042089 3534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: This master's current role is: FOLLOWER
07:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.042207 3540 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85 [sys.catalog]: This master's current role is: FOLLOWER
07:09:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.105728 3550 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:09:09.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.238206 3506 catalog_manager.cc:1259] Loaded cluster ID: 8840025a8e804e269c1ed90a4d837e4f
07:09:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.238724 3506 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: loading cluster ID for follower catalog manager: success
07:09:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.246551 3506 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: acquiring CA information for follower catalog manager: success
07:09:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.252528 3506 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 13ad034f1c404d6f9cbc29366752b96a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:09:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.256345 3531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.256891 3531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.257351 3531 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:09:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.257761 3531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.291440 3531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.292261 3531 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:09:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.292510 3531 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:09:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.293056 3531 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.45.1
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.328366 3531 tablet_server_runner.cc:78] Tablet server non-default flags:
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:09.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.1:0
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data/info.pb
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.1
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.1
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/logs
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.329913 3531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.331713 3531 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.353890 3557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.525449 3562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6149d28a1f6a434ab0dd88551148c905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6149d28a1f6a434ab0dd88551148c905" member_type: VOTER last_known_addr { host: "127.3.45.62" port: 33155 } } peers { permanent_uuid: "9ec4db5ea9874f88b2804e7240700d85" member_type: VOTER last_known_addr { host: "127.3.45.61" port: 42905 } } peers { permanent_uuid: "13ad034f1c404d6f9cbc29366752b96a" member_type: VOTER last_known_addr { host: "127.3.45.60" port: 44303 } } }
07:09:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:09.532797 3562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6149d28a1f6a434ab0dd88551148c905 [sys.catalog]: This master's current role is: LEADER
07:09:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:10.129305 3550 catalog_manager.cc:1259] Loaded cluster ID: 8840025a8e804e269c1ed90a4d837e4f
07:09:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:10.130600 3550 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: loading cluster ID for follower catalog manager: success
07:09:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:10.167459 3550 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: acquiring CA information for follower catalog manager: success
07:09:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:10.209278 3550 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9ec4db5ea9874f88b2804e7240700d85: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:09:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:10.749259 3555 debug-util.cc:398] Leaking SignalData structure 0x7b0800014ea0 after lost signal to thread 3531
07:09:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:11.790187 3531 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.430s user 0.623s sys 1.160s
07:09:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:11.790513 3531 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.430s user 0.623s sys 1.160s
07:09:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:09.360455 3556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:11.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:11.793835 3559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:11.800457 3558 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2431 milliseconds
07:09:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.800477 3531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:09:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.802230 3531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.811029 3531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.812922 3531 hybrid_clock.cc:648] HybridClock initialized: now 1714115351812849 us; error 89 us; skew 500 ppm
07:09:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.814177 3531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:11.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.848656 3531 webserver.cc:460] Webserver started at http://127.3.45.1:35599/ using document root <none> and password file <none>
07:09:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.850346 3531 fs_manager.cc:359] Metadata directory not provided
07:09:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.850778 3531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.851430 3531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.859046 3531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data/instance:
07:09:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "322a33ac71e2422bbfa77c23587bc94f"
07:09:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:11 on dist-test-slave-hqrx"
07:09:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.861006 3531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal/instance:
07:09:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "322a33ac71e2422bbfa77c23587bc94f"
07:09:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:11 on dist-test-slave-hqrx"
07:09:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.886261 3531 fs_manager.cc:691] Time spent creating directory manager: real 0.024s user 0.012s sys 0.012s
07:09:11.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.901440 3570 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:11.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.903139 3531 fs_manager.cc:725] Time spent opening block manager: real 0.012s user 0.004s sys 0.001s
07:09:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.903723 3531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal
07:09:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "322a33ac71e2422bbfa77c23587bc94f"
07:09:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:11 on dist-test-slave-hqrx"
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:11.904388 3531 fs_report.cc:383] FS layout report
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data/data
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:12.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.003018 3531 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.005236 3531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.005942 3531 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.015128 3531 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:09:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.027626 3531 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:09:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.028093 3531 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:09:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.028498 3531 ts_tablet_manager.cc:590] Registered 0 tablets
07:09:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.028826 3531 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:09:12.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.453903 3531 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.1:35709
07:09:12.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.454093 3682 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.1:35709 every 8 connection(s)
07:09:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.456593 3531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/data/info.pb
07:09:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.460134 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3531
07:09:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.460597 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-0/wal/instance
07:09:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.478374 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/logs
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data/info.pb
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.2:0
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.2
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.2
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:12.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:09:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.526036 3684 heartbeater.cc:347] Connected to a master server at 127.3.45.60:44303
07:09:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.526486 3684 heartbeater.cc:464] Registering TS with master...
07:09:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.527849 3684 heartbeater.cc:511] Master 127.3.45.60:44303 requested a full tablet report, sending...
07:09:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.529523 3683 heartbeater.cc:347] Connected to a master server at 127.3.45.61:42905
07:09:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.529906 3683 heartbeater.cc:464] Registering TS with master...
07:09:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.531010 3683 heartbeater.cc:511] Master 127.3.45.61:42905 requested a full tablet report, sending...
07:09:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.531641 3451 ts_manager.cc:194] Registered new tserver with Master: 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709)
07:09:12.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.536875 3372 ts_manager.cc:194] Registered new tserver with Master: 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709)
07:09:12.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.538318 3685 heartbeater.cc:347] Connected to a master server at 127.3.45.62:33155
07:09:12.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.538681 3685 heartbeater.cc:464] Registering TS with master...
07:09:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.539578 3685 heartbeater.cc:511] Master 127.3.45.62:33155 requested a full tablet report, sending...
07:09:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.542218 3297 ts_manager.cc:194] Registered new tserver with Master: 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709)
07:09:12.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.545015 3297 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.45.1:60777
07:09:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.834861 3693 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.835358 3693 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.835741 3693 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:09:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.836138 3693 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.871754 3693 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.872484 3693 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:09:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.872696 3693 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:09:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.873225 3693 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.45.2
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.910055 3693 tablet_server_runner.cc:78] Tablet server non-default flags:
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data
07:09:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.2:0
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data/info.pb
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.2
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.2
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/logs
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.911237 3693 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:12.912926 3693 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:12.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.932485 3701 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:13.549136 3685 heartbeater.cc:503] Master 127.3.45.62:33155 was elected leader, sending a full tablet report...
07:09:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.933593 3703 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:12.936103 3700 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:14.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.125357 3702 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1182 milliseconds
07:09:14.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.125497 3693 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:09:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.126757 3693 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:14.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.129220 3693 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:14.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.130592 3693 hybrid_clock.cc:648] HybridClock initialized: now 1714115354130543 us; error 58 us; skew 500 ppm
07:09:14.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.131498 3693 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.138576 3693 webserver.cc:460] Webserver started at http://127.3.45.2:43429/ using document root <none> and password file <none>
07:09:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.139523 3693 fs_manager.cc:359] Metadata directory not provided
07:09:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.139741 3693 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.140242 3693 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.145036 3693 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data/instance:
07:09:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a8646389e9548a0b1bbe4317c34a69f"
07:09:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:14 on dist-test-slave-hqrx"
07:09:14.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.146251 3693 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal/instance:
07:09:14.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a8646389e9548a0b1bbe4317c34a69f"
07:09:14.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:14 on dist-test-slave-hqrx"
07:09:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.153810 3693 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
07:09:14.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.159322 3711 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.160395 3693 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.160722 3693 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a8646389e9548a0b1bbe4317c34a69f"
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:14 on dist-test-slave-hqrx"
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.161063 3693 fs_report.cc:383] FS layout report
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data/data
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.224488 3693 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.225977 3693 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.226501 3693 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.229321 3693 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:09:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.233546 3693 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:09:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.233759 3693 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:09:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.234046 3693 ts_tablet_manager.cc:590] Registered 0 tablets
07:09:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.234217 3693 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:09:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.373018 3693 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.2:44661
07:09:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.373129 3823 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.2:44661 every 8 connection(s)
07:09:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.376524 3693 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/data/info.pb
07:09:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.379258 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3693
07:09:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.379956 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-1/wal/instance
07:09:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.418414 3252 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/logs
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data/info.pb
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.3:0
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.3
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.3
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false with env {}
07:09:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.437595 3825 heartbeater.cc:347] Connected to a master server at 127.3.45.60:44303
07:09:14.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.438133 3825 heartbeater.cc:464] Registering TS with master...
07:09:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.439458 3825 heartbeater.cc:511] Master 127.3.45.60:44303 requested a full tablet report, sending...
07:09:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.442775 3451 ts_manager.cc:194] Registered new tserver with Master: 9a8646389e9548a0b1bbe4317c34a69f (127.3.45.2:44661)
07:09:14.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.450256 3827 heartbeater.cc:347] Connected to a master server at 127.3.45.62:33155
07:09:14.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.450596 3827 heartbeater.cc:464] Registering TS with master...
07:09:14.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.451553 3827 heartbeater.cc:511] Master 127.3.45.62:33155 requested a full tablet report, sending...
07:09:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.452836 3824 heartbeater.cc:347] Connected to a master server at 127.3.45.61:42905
07:09:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.453181 3824 heartbeater.cc:464] Registering TS with master...
07:09:14.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.454077 3824 heartbeater.cc:511] Master 127.3.45.61:42905 requested a full tablet report, sending...
07:09:14.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.454144 3297 ts_manager.cc:194] Registered new tserver with Master: 9a8646389e9548a0b1bbe4317c34a69f (127.3.45.2:44661)
07:09:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.455986 3297 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.45.2:59745
07:09:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.456719 3372 ts_manager.cc:194] Registered new tserver with Master: 9a8646389e9548a0b1bbe4317c34a69f (127.3.45.2:44661)
07:09:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.762315 3834 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:09:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.762872 3834 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:09:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.763375 3834 flags.cc:425] Enabled unsafe flag: --enable_rowset_compaction=false
07:09:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.763783 3834 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:09:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.794405 3834 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.795140 3834 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
07:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.795411 3834 flags.cc:425] Enabled experimental flag: --flush_threshold_secs=1
07:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.795984 3834 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.45.3
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.831259 3834 tablet_server_runner.cc:78] Tablet server non-default flags:
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.45.20:37667
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal
07:09:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.45.3:0
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data/info.pb
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.45.3
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_rowset_compaction=false
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_mb=1
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --flush_threshold_secs=1
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.45.62:33155,127.3.45.61:42905,127.3.45.60:44303
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.45.3
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/logs
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:09:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:09:14.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.832827 3834 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:09:14.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:14.834676 3834 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:09:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.851203 3840 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:15.459725 3827 heartbeater.cc:503] Master 127.3.45.62:33155 was elected leader, sending a full tablet report...
07:09:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.852310 3843 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:14.853732 3841 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:09:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:16.048280 3842 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1194 milliseconds
07:09:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.048400 3834 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.049695 3834 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:09:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.052623 3834 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:09:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.054059 3834 hybrid_clock.cc:648] HybridClock initialized: now 1714115356053994 us; error 80 us; skew 500 ppm
07:09:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.054852 3834 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:09:16.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.061074 3834 webserver.cc:460] Webserver started at http://127.3.45.3:43931/ using document root <none> and password file <none>
07:09:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.061993 3834 fs_manager.cc:359] Metadata directory not provided
07:09:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.062182 3834 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:09:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.062572 3834 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:09:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.067399 3834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data/instance:
07:09:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd8f057891ad4388bf6ce94c7f9bec71"
07:09:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:16 on dist-test-slave-hqrx"
07:09:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.068526 3834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal/instance:
07:09:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd8f057891ad4388bf6ce94c7f9bec71"
07:09:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:16 on dist-test-slave-hqrx"
07:09:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.075486 3834 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
07:09:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.080964 3851 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:09:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.082019 3834 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
07:09:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.082314 3834 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data,/tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal
07:09:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd8f057891ad4388bf6ce94c7f9bec71"
07:09:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:09:16 on dist-test-slave-hqrx"
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.082628 3834 fs_report.cc:383] FS layout report
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data/data
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:09:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:09:16.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.148106 3834 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:09:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.149663 3834 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:09:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.150254 3834 kserver.cc:163] Server-wide thread pool size limit: 3276
07:09:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.152935 3834 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:09:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.157070 3834 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:09:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.157259 3834 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:09:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.157456 3834 ts_tablet_manager.cc:590] Registered 0 tablets
07:09:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.157601 3834 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:09:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.295814 3834 rpc_server.cc:307] RPC server started. Bound to: 127.3.45.3:43935
07:09:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.295969 3963 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.45.3:43935 every 8 connection(s)
07:09:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.298282 3834 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/data/info.pb
07:09:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.301716 3252 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu as pid 3834
07:09:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.302206 3252 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRfRc_3/test-tmp/mini-kudu-cluster8885085288262038768/ts-2/wal/instance
07:09:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.342861 3966 heartbeater.cc:347] Connected to a master server at 127.3.45.62:33155
07:09:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.343425 3966 heartbeater.cc:464] Registering TS with master...
07:09:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.344794 3966 heartbeater.cc:511] Master 127.3.45.62:33155 requested a full tablet report, sending...
07:09:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.345250 3965 heartbeater.cc:347] Connected to a master server at 127.3.45.60:44303
07:09:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.345525 3965 heartbeater.cc:464] Registering TS with master...
07:09:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.346252 3965 heartbeater.cc:511] Master 127.3.45.60:44303 requested a full tablet report, sending...
07:09:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.347306 3297 ts_manager.cc:194] Registered new tserver with Master: bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935)
07:09:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.348217 3964 heartbeater.cc:347] Connected to a master server at 127.3.45.61:42905
07:09:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.348551 3964 heartbeater.cc:464] Registering TS with master...
07:09:16.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.348852 3451 ts_manager.cc:194] Registered new tserver with Master: bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935)
07:09:16.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.349033 3297 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.45.3:49121
07:09:16.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.349449 3964 heartbeater.cc:511] Master 127.3.45.61:42905 requested a full tablet report, sending...
07:09:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.351912 3372 ts_manager.cc:194] Registered new tserver with Master: bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935)
07:09:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.368916 3252 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:09:16.382 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:09:16.382 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:09:16.384 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.45.62"
port: 33155
}
bound_http_address {
host: "127.3.45.62"
port: 45615
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.45.61"
port: 42905
}
bound_http_address {
host: "127.3.45.61"
port: 32817
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.45.60"
port: 44303
}
bound_http_address {
host: "127.3.45.60"
port: 35079
}
}
}
07:09:16.384 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:09:16.385 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.45.1"
port: 35709
}
bound_http_address {
host: "127.3.45.1"
port: 35599
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.45.2"
port: 44661
}
bound_http_address {
host: "127.3.45.2"
port: 43429
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.45.3"
port: 43935
}
bound_http_address {
host: "127.3.45.3"
port: 43931
}
}
}
07:09:16.385 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:09:16.387 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:09:16.388 [INFO - main] (Logging.scala:57) Running Spark version 3.1.2
07:09:16.389 [INFO - main] (Logging.scala:57) ==============================================================
07:09:16.389 [INFO - main] (Logging.scala:57) No custom resources configured for spark.driver.
07:09:16.389 [INFO - main] (Logging.scala:57) ==============================================================
07:09:16.389 [INFO - main] (Logging.scala:57) Submitted application: test
07:09:16.390 [INFO - main] (Logging.scala:57) Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
07:09:16.390 [INFO - main] (Logging.scala:57) Limiting resource is cpu
07:09:16.390 [INFO - main] (Logging.scala:57) Added ResourceProfile id: 0
07:09:16.391 [INFO - main] (Logging.scala:57) Changing view acls to: slave
07:09:16.392 [INFO - main] (Logging.scala:57) Changing modify acls to: slave
07:09:16.392 [INFO - main] (Logging.scala:57) Changing view acls groups to:
07:09:16.392 [INFO - main] (Logging.scala:57) Changing modify acls groups to:
07:09:16.392 [INFO - main] (Logging.scala:57) SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(slave); groups with view permissions: Set(); users with modify permissions: Set(slave); groups with modify permissions: Set()
07:09:16.415 [INFO - main] (Logging.scala:57) Successfully started service 'sparkDriver' on port 35001.
07:09:16.419 [INFO - main] (Logging.scala:57) Registering MapOutputTracker
07:09:16.420 [INFO - main] (Logging.scala:57) Registering BlockManagerMaster
07:09:16.420 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
07:09:16.420 [INFO - main] (Logging.scala:57) BlockManagerMasterEndpoint up
07:09:16.421 [INFO - main] (Logging.scala:57) Registering BlockManagerMasterHeartbeat
07:09:16.422 [INFO - main] (Logging.scala:57) Created local directory at /tmp/blockmgr-b86105ca-3975-4a53-94c5-40bda5119bb7
07:09:16.423 [INFO - main] (Logging.scala:57) MemoryStore started with capacity 1993.9 MiB
07:09:16.424 [INFO - main] (Logging.scala:57) Registering OutputCommitCoordinator
07:09:16.449 [INFO - main] (Logging.scala:57) Starting executor ID driver on host dist-test-slave-hqrx.c.gcp-upstream.internal
07:09:16.453 [INFO - main] (Logging.scala:57) Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 37193.
07:09:16.454 [INFO - main] (NettyBlockTransferService.scala:81) Server created on dist-test-slave-hqrx.c.gcp-upstream.internal:37193
07:09:16.454 [INFO - main] (Logging.scala:57) Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
07:09:16.454 [INFO - main] (Logging.scala:57) Registering BlockManager BlockManagerId(driver, dist-test-slave-hqrx.c.gcp-upstream.internal, 37193, None)
07:09:16.454 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Registering block manager dist-test-slave-hqrx.c.gcp-upstream.internal:37193 with 1993.9 MiB RAM, BlockManagerId(driver, dist-test-slave-hqrx.c.gcp-upstream.internal, 37193, None)
07:09:16.455 [INFO - main] (Logging.scala:57) Registered BlockManager BlockManagerId(driver, dist-test-slave-hqrx.c.gcp-upstream.internal, 37193, None)
07:09:16.456 [INFO - main] (Logging.scala:57) Initialized BlockManager: BlockManagerId(driver, dist-test-slave-hqrx.c.gcp-upstream.internal, 37193, None)
07:09:16.461 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:09:16.470 [DEBUG - main] (Connection.java:748) [peer master-127.3.45.62:33155(127.3.45.62:33155)] connecting to peer
07:09:16.471 [DEBUG - main] (Connection.java:748) [peer master-127.3.45.61:42905(127.3.45.61:42905)] connecting to peer
07:09:16.476 [DEBUG - main] (Connection.java:748) [peer master-127.3.45.60:44303(127.3.45.60:44303)] connecting to peer
07:09:16.479 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.45.62:33155(127.3.45.62:33155)] Successfully connected to peer
07:09:16.480 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.45.60:44303(127.3.45.60:44303)] Successfully connected to peer
07:09:16.480 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.45.61:42905(127.3.45.61:42905)] Successfully connected to peer
07:09:16.481 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.45.62
07:09:16.485 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.45.60
07:09:16.489 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.45.61
07:09:16.493 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:16.497 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:16.498 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:16.499 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdd829e71, L:/127.0.0.1:54996 - R:/127.3.45.62:33155] using SASL/PLAIN
07:09:16.501 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0ec091c1, L:/127.0.0.1:52278 - R:/127.3.45.60:44303] using SASL/PLAIN
07:09:16.502 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa0501133, L:/127.0.0.1:58346 - R:/127.3.45.61:42905] using SASL/PLAIN
07:09:16.503 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.507066 3297 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54996:
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "test"
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c1_i"
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c2_s"
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c3_double"
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DOUBLE
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c4_long"
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c5_bool"
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c6_short"
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT16
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c7_float"
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: FLOAT
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c8_binary"
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BINARY
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c9_unixtime_micros"
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: UNIXTIME_MICROS
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c10_byte"
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c11_decimal32"
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL32
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 9
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c12_decimal64"
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL64
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 18
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c13_decimal128"
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL128
07:09:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 38
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c14_varchar"
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: VARCHAR
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) length: 65535
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c15_date"
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DATE
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\000\000\000\000\007\001\000\000\0002\000\000\000\006\001\000\000\0002\000\000\000\007\000\000\000\000"
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) owner: "testuser"
07:09:16.545 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table e8cfaebe4903455a94e977ba9c6d6928
07:09:16.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.580961 3618 tablet_service.cc:1467] Processing CreateTablet for tablet eee82d6b42a949709d791678f82e3d2b (DEFAULT_TABLE table=test [id=e8cfaebe4903455a94e977ba9c6d6928]), partition=RANGE (key) PARTITION VALUES < 50
07:09:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.583218 3618 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eee82d6b42a949709d791678f82e3d2b. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.593406 3759 tablet_service.cc:1467] Processing CreateTablet for tablet 499c145eb0f34c41b6196b5647a381b3 (DEFAULT_TABLE table=test [id=e8cfaebe4903455a94e977ba9c6d6928]), partition=RANGE (key) PARTITION 50 <= VALUES
07:09:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.595099 3759 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 499c145eb0f34c41b6196b5647a381b3. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:16.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.604936 4019 tablet_bootstrap.cc:492] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: Bootstrap starting.
07:09:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.610141 4019 tablet_bootstrap.cc:654] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: Neither blocks nor log segments found. Creating new log.
07:09:16.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.612226 4019 log.cc:824] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: Log is configured to *not* fsync() on all Append() calls
07:09:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.617096 4021 tablet_bootstrap.cc:492] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: Bootstrap starting.
07:09:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.617252 4019 tablet_bootstrap.cc:492] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: No bootstrap required, opened a new log
07:09:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.617764 4019 ts_tablet_manager.cc:1367] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: Time spent bootstrapping tablet: real 0.013s user 0.009s sys 0.004s
07:09:16.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.622897 4021 tablet_bootstrap.cc:654] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: Neither blocks nor log segments found. Creating new log.
07:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.624876 4021 log.cc:824] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: Log is configured to *not* fsync() on all Append() calls
07:09:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.629652 4021 tablet_bootstrap.cc:492] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: No bootstrap required, opened a new log
07:09:16.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.630005 4021 ts_tablet_manager.cc:1367] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: Time spent bootstrapping tablet: real 0.013s user 0.007s sys 0.004s
07:09:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.640193 4019 raft_consensus.cc:348] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } }
07:09:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.640798 4019 raft_consensus.cc:374] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.641080 4019 raft_consensus.cc:729] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 322a33ac71e2422bbfa77c23587bc94f, State: Initialized, Role: FOLLOWER
07:09:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.641945 4019 consensus_queue.cc:260] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } }
07:09:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.642513 4019 raft_consensus.cc:388] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:09:16.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.642820 4019 raft_consensus.cc:482] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:09:16.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.643170 4019 raft_consensus.cc:3037] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Advancing to term 1
07:09:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.647279 4019 raft_consensus.cc:504] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } }
07:09:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.648058 4019 leader_election.cc:302] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 322a33ac71e2422bbfa77c23587bc94f; no voters:
07:09:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.650310 4019 leader_election.cc:288] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [CANDIDATE]: Term 1 election: Requested vote from peers
07:09:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.650910 4023 raft_consensus.cc:2781] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 1 FOLLOWER]: Leader election won for term 1
07:09:16.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.651923 4021 raft_consensus.cc:348] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } }
07:09:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.653746 4021 raft_consensus.cc:374] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.654101 4021 raft_consensus.cc:729] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a8646389e9548a0b1bbe4317c34a69f, State: Initialized, Role: FOLLOWER
07:09:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.655746 4021 consensus_queue.cc:260] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } }
07:09:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.656354 4021 raft_consensus.cc:388] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:09:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.656641 4021 raft_consensus.cc:482] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:09:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.656906 4019 ts_tablet_manager.cc:1398] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f: Time spent starting tablet: real 0.039s user 0.025s sys 0.012s
07:09:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.657022 4021 raft_consensus.cc:3037] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Advancing to term 1
07:09:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.658486 4023 raft_consensus.cc:686] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [term 1 LEADER]: Becoming Leader. State: Replica: 322a33ac71e2422bbfa77c23587bc94f, State: Running, Role: LEADER
07:09:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.659250 4023 consensus_queue.cc:237] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } }
07:09:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.664199 4021 raft_consensus.cc:504] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } }
07:09:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.665091 4021 leader_election.cc:302] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 9a8646389e9548a0b1bbe4317c34a69f; no voters:
07:09:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.672865 4021 leader_election.cc:288] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 election: Requested vote from peers
07:09:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.674698 4027 raft_consensus.cc:2781] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 FOLLOWER]: Leader election won for term 1
07:09:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.680611 4021 ts_tablet_manager.cc:1398] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f: Time spent starting tablet: real 0.050s user 0.036s sys 0.008s
07:09:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.681816 4027 raft_consensus.cc:686] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 LEADER]: Becoming Leader. State: Replica: 9a8646389e9548a0b1bbe4317c34a69f, State: Running, Role: LEADER
07:09:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.682823 4027 consensus_queue.cc:237] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } }
07:09:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.679365 3297 catalog_manager.cc:5484] T eee82d6b42a949709d791678f82e3d2b P 322a33ac71e2422bbfa77c23587bc94f reported cstate change: term changed from 0 to 1, leader changed from <none> to 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1). New cstate: current_term: 1 leader_uuid: "322a33ac71e2422bbfa77c23587bc94f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } health_report { overall_health: HEALTHY } } }
07:09:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.717579 3296 catalog_manager.cc:5484] T 499c145eb0f34c41b6196b5647a381b3 P 9a8646389e9548a0b1bbe4317c34a69f reported cstate change: term changed from 0 to 1, leader changed from <none> to 9a8646389e9548a0b1bbe4317c34a69f (127.3.45.2). New cstate: current_term: 1 leader_uuid: "9a8646389e9548a0b1bbe4317c34a69f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } health_report { overall_health: HEALTHY } } }
07:09:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:16.740984 3692 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:eee82d6b42a949709d791678f82e3d2b)
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.814050 3296 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54996:
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "simple-test"
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "val"
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 1
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.832 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 73a907338fe84f09a4cb626c37246986
07:09:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.855844 3899 tablet_service.cc:1467] Processing CreateTablet for tablet 81f5e0d1d4064f51bef5ebad09e75a46 (DEFAULT_TABLE table=simple-test [id=73a907338fe84f09a4cb626c37246986]), partition=RANGE (key) PARTITION UNBOUNDED
07:09:16.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.857273 3899 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81f5e0d1d4064f51bef5ebad09e75a46. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:16.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.877041 4036 tablet_bootstrap.cc:492] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: Bootstrap starting.
07:09:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.881729 4036 tablet_bootstrap.cc:654] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: Neither blocks nor log segments found. Creating new log.
07:09:16.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.883368 4036 log.cc:824] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: Log is configured to *not* fsync() on all Append() calls
07:09:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.887667 4036 tablet_bootstrap.cc:492] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: No bootstrap required, opened a new log
07:09:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.888134 4036 ts_tablet_manager.cc:1367] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: Time spent bootstrapping tablet: real 0.012s user 0.011s sys 0.000s
07:09:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.908470 4036 raft_consensus.cc:348] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.909065 4036 raft_consensus.cc:374] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.909243 4036 raft_consensus.cc:729] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bd8f057891ad4388bf6ce94c7f9bec71, State: Initialized, Role: FOLLOWER
07:09:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:16.910020 3831 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:499c145eb0f34c41b6196b5647a381b3)
07:09:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.909871 4036 consensus_queue.cc:260] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.910355 4036 raft_consensus.cc:388] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
07:09:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.910602 4036 raft_consensus.cc:482] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
07:09:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.910877 4036 raft_consensus.cc:3037] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Advancing to term 1
07:09:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.915450 4036 raft_consensus.cc:504] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:16.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.916229 4036 leader_election.cc:302] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: bd8f057891ad4388bf6ce94c7f9bec71; no voters:
07:09:16.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.918241 4036 leader_election.cc:288] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [CANDIDATE]: Term 1 election: Requested vote from peers
07:09:16.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.918596 4038 raft_consensus.cc:2781] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 1 FOLLOWER]: Leader election won for term 1
07:09:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.920770 4038 raft_consensus.cc:686] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [term 1 LEADER]: Becoming Leader. State: Replica: bd8f057891ad4388bf6ce94c7f9bec71, State: Running, Role: LEADER
07:09:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.921610 3966 heartbeater.cc:503] Master 127.3.45.62:33155 was elected leader, sending a full tablet report...
07:09:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.922209 4036 ts_tablet_manager.cc:1398] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71: Time spent starting tablet: real 0.034s user 0.030s sys 0.002s
07:09:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.921916 4038 consensus_queue.cc:237] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:16.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.938068 3296 catalog_manager.cc:5484] T 81f5e0d1d4064f51bef5ebad09e75a46 P bd8f057891ad4388bf6ce94c7f9bec71 reported cstate change: term changed from 0 to 1, leader changed from <none> to bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3). New cstate: current_term: 1 leader_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } health_report { overall_health: HEALTHY } } }
07:09:16.952 [INFO - main] (RandomUtils.java:49) Using random seed: 1714115356952
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.954857 3296 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54996:
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "split-size-table"
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c1_i"
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c2_s"
07:09:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c3_double"
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DOUBLE
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c4_long"
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c5_bool"
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c6_short"
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT16
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c7_float"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: FLOAT
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c8_binary"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BINARY
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c9_unixtime_micros"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: UNIXTIME_MICROS
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c10_byte"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c11_decimal32"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL32
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 9
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c12_decimal64"
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL64
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 18
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c13_decimal128"
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DECIMAL128
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) precision: 38
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c14_varchar"
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: VARCHAR
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type_attributes {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) length: 65535
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "c15_date"
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: DATE
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:09:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:16.958282 3296 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table split-size-table in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:09:16.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a8de415057434381a66071c0391915f0
07:09:16.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.988752 3618 tablet_service.cc:1467] Processing CreateTablet for tablet cc690e52d1e344379ce58aa35abf0e16 (DEFAULT_TABLE table=split-size-table [id=a8de415057434381a66071c0391915f0]), partition=RANGE (key) PARTITION UNBOUNDED
07:09:16.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.989977 3618 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc690e52d1e344379ce58aa35abf0e16. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:16.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.990481 3759 tablet_service.cc:1467] Processing CreateTablet for tablet cc690e52d1e344379ce58aa35abf0e16 (DEFAULT_TABLE table=split-size-table [id=a8de415057434381a66071c0391915f0]), partition=RANGE (key) PARTITION UNBOUNDED
07:09:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.991672 3759 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc690e52d1e344379ce58aa35abf0e16. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.991432 3899 tablet_service.cc:1467] Processing CreateTablet for tablet cc690e52d1e344379ce58aa35abf0e16 (DEFAULT_TABLE table=split-size-table [id=a8de415057434381a66071c0391915f0]), partition=RANGE (key) PARTITION UNBOUNDED
07:09:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:16.992566 3899 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc690e52d1e344379ce58aa35abf0e16. 1 dirs total, 0 dirs full, 0 dirs failed
07:09:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.002985 4036 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71: Bootstrap starting.
07:09:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.005210 4019 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f: Bootstrap starting.
07:09:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.005920 4021 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Bootstrap starting.
07:09:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.008893 4036 tablet_bootstrap.cc:654] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71: Neither blocks nor log segments found. Creating new log.
07:09:17.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.010260 4021 tablet_bootstrap.cc:654] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Neither blocks nor log segments found. Creating new log.
07:09:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.011169 4019 tablet_bootstrap.cc:654] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f: Neither blocks nor log segments found. Creating new log.
07:09:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.019423 4036 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71: No bootstrap required, opened a new log
07:09:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.019493 4021 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: No bootstrap required, opened a new log
07:09:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.019698 4019 tablet_bootstrap.cc:492] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f: No bootstrap required, opened a new log
07:09:17.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.019793 4036 ts_tablet_manager.cc:1367] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71: Time spent bootstrapping tablet: real 0.017s user 0.009s sys 0.004s
07:09:17.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.020105 4019 ts_tablet_manager.cc:1367] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f: Time spent bootstrapping tablet: real 0.015s user 0.007s sys 0.007s
07:09:17.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.020184 4021 ts_tablet_manager.cc:1367] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.003s
07:09:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.021925 4036 raft_consensus.cc:348] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.022533 4036 raft_consensus.cc:374] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.022831 4036 raft_consensus.cc:729] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bd8f057891ad4388bf6ce94c7f9bec71, State: Initialized, Role: FOLLOWER
07:09:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.023053 4021 raft_consensus.cc:348] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.023770 4021 raft_consensus.cc:374] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.023459 4036 consensus_queue.cc:260] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.024062 4021 raft_consensus.cc:729] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a8646389e9548a0b1bbe4317c34a69f, State: Initialized, Role: FOLLOWER
07:09:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.023699 4019 raft_consensus.cc:348] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.024421 4019 raft_consensus.cc:374] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:09:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.024693 4019 raft_consensus.cc:729] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 322a33ac71e2422bbfa77c23587bc94f, State: Initialized, Role: FOLLOWER
07:09:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.025480 4036 ts_tablet_manager.cc:1398] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
07:09:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.025310 4019 consensus_queue.cc:260] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.027586 4019 ts_tablet_manager.cc:1398] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
07:09:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.036311 4021 consensus_queue.cc:260] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.041090 4021 ts_tablet_manager.cc:1398] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Time spent starting tablet: real 0.021s user 0.000s sys 0.007s
07:09:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:17.077764 3971 tablet_mm_ops.cc:140] Rowset compaction is disabled (check --enable_rowset_compaction and disable_compaction in extra_config for tablet:81f5e0d1d4064f51bef5ebad09e75a46)
07:09:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.310942 4027 raft_consensus.cc:482] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:09:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.311496 4027 raft_consensus.cc:504] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.315475 4027 leader_election.cc:288] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709), bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935)
07:09:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.328691 3919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc690e52d1e344379ce58aa35abf0e16" candidate_uuid: "9a8646389e9548a0b1bbe4317c34a69f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" is_pre_election: true
07:09:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.329610 3919 raft_consensus.cc:2454] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a8646389e9548a0b1bbe4317c34a69f in term 0.
07:09:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.330039 3638 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc690e52d1e344379ce58aa35abf0e16" candidate_uuid: "9a8646389e9548a0b1bbe4317c34a69f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "322a33ac71e2422bbfa77c23587bc94f" is_pre_election: true
07:09:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.330907 3638 raft_consensus.cc:2454] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a8646389e9548a0b1bbe4317c34a69f in term 0.
07:09:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.331073 3715 leader_election.cc:302] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a8646389e9548a0b1bbe4317c34a69f, bd8f057891ad4388bf6ce94c7f9bec71; no voters:
07:09:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.331965 4027 raft_consensus.cc:2781] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Leader pre-election won for term 1
07:09:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.332327 4027 raft_consensus.cc:482] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:09:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.332685 4027 raft_consensus.cc:3037] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 0 FOLLOWER]: Advancing to term 1
07:09:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.339498 4027 raft_consensus.cc:504] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.341537 4027 leader_election.cc:288] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 election: Requested vote from peers 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709), bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935)
07:09:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.342633 3919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc690e52d1e344379ce58aa35abf0e16" candidate_uuid: "9a8646389e9548a0b1bbe4317c34a69f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bd8f057891ad4388bf6ce94c7f9bec71"
07:09:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.342522 3638 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc690e52d1e344379ce58aa35abf0e16" candidate_uuid: "9a8646389e9548a0b1bbe4317c34a69f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "322a33ac71e2422bbfa77c23587bc94f"
07:09:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.343086 3919 raft_consensus.cc:3037] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 0 FOLLOWER]: Advancing to term 1
07:09:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.343092 3638 raft_consensus.cc:3037] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 0 FOLLOWER]: Advancing to term 1
07:09:17.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.347366 3919 raft_consensus.cc:2454] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a8646389e9548a0b1bbe4317c34a69f in term 1.
07:09:17.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.348281 3715 leader_election.cc:302] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a8646389e9548a0b1bbe4317c34a69f, bd8f057891ad4388bf6ce94c7f9bec71; no voters:
07:09:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.348995 4027 raft_consensus.cc:2781] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 FOLLOWER]: Leader election won for term 1
07:09:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.349735 3638 raft_consensus.cc:2454] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a8646389e9548a0b1bbe4317c34a69f in term 1.
07:09:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.351094 4027 raft_consensus.cc:686] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [term 1 LEADER]: Becoming Leader. State: Replica: 9a8646389e9548a0b1bbe4317c34a69f, State: Running, Role: LEADER
07:09:17.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.352198 4027 consensus_queue.cc:237] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } }
07:09:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.362149 3297 catalog_manager.cc:5484] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f reported cstate change: term changed from 0 to 1, leader changed from <none> to 9a8646389e9548a0b1bbe4317c34a69f (127.3.45.2). New cstate: current_term: 1 leader_uuid: "9a8646389e9548a0b1bbe4317c34a69f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9a8646389e9548a0b1bbe4317c34a69f" member_type: VOTER last_known_addr { host: "127.3.45.2" port: 44661 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 } health_report { overall_health: UNKNOWN } } }
07:09:17.642 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:09:17.642 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:09:17.642 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=null) after lookup
07:09:17.643 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:09:17.645 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] Successfully connected to peer
07:09:17.655 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:17.658 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc317261f, L:/127.0.0.1:49968 - R:/127.3.45.2:44661] using TOKEN/null
07:09:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.675671 3919 raft_consensus.cc:1261] T cc690e52d1e344379ce58aa35abf0e16 P bd8f057891ad4388bf6ce94c7f9bec71 [term 1 FOLLOWER]: Refusing update from remote peer 9a8646389e9548a0b1bbe4317c34a69f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:09:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.675808 3638 raft_consensus.cc:1261] T cc690e52d1e344379ce58aa35abf0e16 P 322a33ac71e2422bbfa77c23587bc94f [term 1 FOLLOWER]: Refusing update from remote peer 9a8646389e9548a0b1bbe4317c34a69f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:09:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.677268 4049 consensus_queue.cc:1035] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [LEADER]: Connected to new peer: Peer: permanent_uuid: "bd8f057891ad4388bf6ce94c7f9bec71" member_type: VOTER last_known_addr { host: "127.3.45.3" port: 43935 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:09:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.677942 4027 consensus_queue.cc:1035] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f [LEADER]: Connected to new peer: Peer: permanent_uuid: "322a33ac71e2422bbfa77c23587bc94f" member_type: VOTER last_known_addr { host: "127.3.45.1" port: 35709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:09:17.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.714720 4052 mvcc.cc:205] Tried to move back new op lower bound from 7021016505022009344 to 7021016503723794432. Current Snapshot: MvccSnapshot[applied={T|T < 7021016505022009344}]
07:09:17.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:17.719412 4055 mvcc.cc:205] Tried to move back new op lower bound from 7021016505022009344 to 7021016503723794432. Current Snapshot: MvccSnapshot[applied={T|T < 7021016505022009344}]
07:09:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.082442 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969062
07:09:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.167583 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969062
07:09:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.248742 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969062
07:09:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.417847 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.334s user 0.148s sys 0.031s Metrics: {"bytes_written":39287,"cfile_init":1,"compiler_manager_pool.queue_time_us":2156,"dirs.queue_time_us":6940,"dirs.run_cpu_time_us":321,"dirs.run_wall_time_us":5514,"drs_written":1,"lbm_read_time_us":250,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8265,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":17,"thread_start_us":9051,"threads_started":2}
07:09:18.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.421586 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 331 bytes on disk
07:09:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.476608 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.005s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":286,"lbm_reads_lt_1ms":4}
07:09:18.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.537675 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.369s user 0.149s sys 0.049s Metrics: {"bytes_written":39308,"cfile_init":1,"compiler_manager_pool.queue_time_us":21019,"dirs.queue_time_us":11223,"dirs.run_cpu_time_us":339,"dirs.run_wall_time_us":1885,"drs_written":1,"lbm_read_time_us":301,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6514,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":21,"spinlock_wait_cycles":28937600,"thread_start_us":30527,"threads_started":2}
07:09:18.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.540664 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 372 bytes on disk
07:09:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.542595 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":280,"lbm_reads_lt_1ms":4}
07:09:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.891043 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.641s user 0.168s sys 0.038s Metrics: {"bytes_written":39314,"cfile_init":1,"compiler_manager_pool.queue_time_us":25541,"dirs.queue_time_us":30677,"dirs.run_cpu_time_us":324,"dirs.run_wall_time_us":2105,"drs_written":1,"lbm_read_time_us":247,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11855,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":23,"thread_start_us":94883,"threads_started":2}
07:09:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.897433 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 392 bytes on disk
07:09:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:18.905220 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":220,"lbm_reads_lt_1ms":4}
07:09:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.407373 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.548810 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:19.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.797210 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.389s user 0.161s sys 0.007s Metrics: {"bytes_written":39506,"cfile_init":1,"dirs.queue_time_us":7084,"dirs.run_cpu_time_us":294,"dirs.run_wall_time_us":9222,"drs_written":1,"lbm_read_time_us":231,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6264,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"spinlock_wait_cycles":3167488,"thread_start_us":7306,"threads_started":1}
07:09:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.799561 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.801517 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":254,"lbm_reads_lt_1ms":4}
07:09:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.814723 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:19.852913 4050 log.cc:924] Time spent T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Append to log took a long time: real 0.056s user 0.001s sys 0.000s
07:09:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.863360 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.314s user 0.137s sys 0.000s Metrics: {"bytes_written":39509,"cfile_init":1,"dirs.queue_time_us":15845,"dirs.run_cpu_time_us":304,"dirs.run_wall_time_us":13739,"drs_written":1,"lbm_read_time_us":245,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6410,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":29,"spinlock_wait_cycles":53120,"thread_start_us":16089,"threads_started":1}
07:09:19.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.876720 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 462 bytes on disk
07:09:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:19.878253 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":232,"lbm_reads_lt_1ms":4}
07:09:20.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:20.103462 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.288s user 0.144s sys 0.022s Metrics: {"bytes_written":39502,"cfile_init":1,"dirs.queue_time_us":6874,"dirs.run_cpu_time_us":274,"dirs.run_wall_time_us":1561,"drs_written":1,"lbm_read_time_us":264,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6524,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28,"thread_start_us":6890,"threads_started":1}
07:09:20.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:20.126423 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:20.128077 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":230,"lbm_reads_lt_1ms":4}
07:09:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:20.911989 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:20.968815 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.135485 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:21.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.198513 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.286s user 0.156s sys 0.003s Metrics: {"bytes_written":39537,"cfile_init":1,"dirs.queue_time_us":10259,"dirs.run_cpu_time_us":305,"dirs.run_wall_time_us":5588,"drs_written":1,"lbm_read_time_us":206,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6431,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28,"thread_start_us":11106,"threads_started":1}
07:09:21.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.201000 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.202745 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":235,"lbm_reads_lt_1ms":4}
07:09:21.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.453078 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.483s user 0.147s sys 0.020s Metrics: {"bytes_written":39564,"cfile_init":1,"dirs.queue_time_us":9780,"dirs.run_cpu_time_us":320,"dirs.run_wall_time_us":7556,"drs_written":1,"lbm_read_time_us":218,"lbm_reads_lt_1ms":4,"lbm_write_time_us":58582,"lbm_writes_1-10_ms":1,"lbm_writes_10-100_ms":2,"lbm_writes_lt_1ms":74,"peak_mem_usage":0,"rows_written":32,"thread_start_us":9996,"threads_started":1}
07:09:21.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.455859 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 493 bytes on disk
07:09:21.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.477277 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":265,"lbm_reads_lt_1ms":4}
07:09:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:21.743321 4050 log.cc:924] Time spent T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f: Append to log took a long time: real 0.059s user 0.002s sys 0.000s
07:09:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.803797 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.668s user 0.161s sys 0.004s Metrics: {"bytes_written":39559,"cfile_init":1,"dirs.queue_time_us":9784,"dirs.run_cpu_time_us":331,"dirs.run_wall_time_us":19483,"drs_written":1,"lbm_read_time_us":273,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6215,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":29424,"threads_started":1}
07:09:21.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.806596 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 483 bytes on disk
07:09:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:21.808506 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":264,"lbm_reads_lt_1ms":4}
07:09:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:22.209754 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969121
07:09:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:22.564050 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:22.572656 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.018389 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.801s user 0.150s sys 0.015s Metrics: {"bytes_written":39544,"cfile_init":1,"dirs.queue_time_us":83770,"dirs.run_cpu_time_us":278,"dirs.run_wall_time_us":9918,"drs_written":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11731,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":26,"thread_start_us":83937,"threads_started":1}
07:09:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.022044 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 427 bytes on disk
07:09:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.036762 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":294,"lbm_reads_lt_1ms":4}
07:09:23.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.371049 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.797s user 0.185s sys 0.027s Metrics: {"bytes_written":39577,"cfile_init":1,"dirs.queue_time_us":164313,"dirs.run_cpu_time_us":334,"dirs.run_wall_time_us":13861,"drs_written":1,"lbm_read_time_us":222,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8928,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":178363,"threads_started":1}
07:09:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.373988 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.395081 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":259,"lbm_reads_lt_1ms":4}
07:09:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.429133 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.863s user 0.153s sys 0.016s Metrics: {"bytes_written":39561,"cfile_init":1,"dirs.queue_time_us":8845,"dirs.run_cpu_time_us":314,"dirs.run_wall_time_us":15109,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":25830,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":28,"thread_start_us":8534,"threads_started":1}
07:09:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.432226 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:23.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.436293 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":278,"lbm_reads_lt_1ms":4}
07:09:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.559535 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:23.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.859587 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.299s user 0.142s sys 0.028s Metrics: {"bytes_written":39581,"cfile_init":1,"dirs.queue_time_us":6857,"dirs.run_cpu_time_us":298,"dirs.run_wall_time_us":1634,"drs_written":1,"lbm_read_time_us":314,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7954,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":28,"spinlock_wait_cycles":493696,"thread_start_us":6839,"threads_started":1}
07:09:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.862383 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.871922 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.001s Metrics: {"cfile_init":1,"lbm_read_time_us":234,"lbm_reads_lt_1ms":4}
07:09:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.916290 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:23.941305 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.186004 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.269s user 0.149s sys 0.013s Metrics: {"bytes_written":39596,"cfile_init":1,"dirs.queue_time_us":7038,"dirs.run_cpu_time_us":287,"dirs.run_wall_time_us":1681,"drs_written":1,"lbm_read_time_us":201,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6472,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":605,"threads_started":1}
07:09:24.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.188757 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:24.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.191056 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4}
07:09:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.260430 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.316s user 0.164s sys 0.001s Metrics: {"bytes_written":39602,"cfile_init":1,"compiler_manager_pool.queue_time_us":210992,"compiler_manager_pool.run_cpu_time_us":302,"compiler_manager_pool.run_wall_time_us":302,"dirs.queue_time_us":2962,"dirs.run_cpu_time_us":317,"dirs.run_wall_time_us":3739,"drs_written":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6306,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":2688,"threads_started":1}
07:09:24.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.262919 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.264843 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4}
07:09:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.634007 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.696463 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.769404 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:24.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.911188 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.275s user 0.181s sys 0.005s Metrics: {"bytes_written":39811,"cfile_init":1,"dirs.queue_time_us":839,"dirs.run_cpu_time_us":299,"dirs.run_wall_time_us":1572,"drs_written":1,"lbm_read_time_us":221,"lbm_reads_lt_1ms":4,"lbm_write_time_us":15700,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":38,"spinlock_wait_cycles":11648,"thread_start_us":543,"threads_started":1}
07:09:24.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.914181 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 561 bytes on disk
07:09:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:24.916415 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":273,"lbm_reads_lt_1ms":4}
07:09:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.016039 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.246s user 0.166s sys 0.000s Metrics: {"bytes_written":39647,"cfile_init":1,"dirs.queue_time_us":1240,"dirs.run_cpu_time_us":296,"dirs.run_wall_time_us":1538,"drs_written":1,"lbm_read_time_us":244,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5872,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"thread_start_us":946,"threads_started":1}
07:09:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.019047 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 491 bytes on disk
07:09:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.024878 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.004s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":309,"lbm_reads_lt_1ms":4}
07:09:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.027721 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.330s user 0.145s sys 0.050s Metrics: {"bytes_written":39642,"cfile_init":1,"dirs.queue_time_us":7015,"dirs.run_cpu_time_us":302,"dirs.run_wall_time_us":3484,"drs_written":1,"lbm_read_time_us":213,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6164,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":7180,"threads_started":1}
07:09:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.032177 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 481 bytes on disk
07:09:25.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.034312 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":247,"lbm_reads_lt_1ms":4}
07:09:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.421574 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:25.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.529788 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.539227 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.759856 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.337s user 0.163s sys 0.002s Metrics: {"bytes_written":39655,"cfile_init":1,"dirs.queue_time_us":11427,"dirs.run_cpu_time_us":294,"dirs.run_wall_time_us":9718,"drs_written":1,"lbm_read_time_us":219,"lbm_reads_lt_1ms":4,"lbm_write_time_us":26311,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":11605,"threads_started":1}
07:09:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.781607 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.783344 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":217,"lbm_reads_lt_1ms":4}
07:09:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.803718 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.273s user 0.160s sys 0.012s Metrics: {"bytes_written":39652,"cfile_init":1,"dirs.queue_time_us":6835,"dirs.run_cpu_time_us":302,"dirs.run_wall_time_us":13919,"drs_written":1,"lbm_read_time_us":218,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5967,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":7041,"threads_started":1}
07:09:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.806696 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.808693 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.269s user 0.174s sys 0.013s Metrics: {"bytes_written":39793,"cfile_init":1,"dirs.queue_time_us":9700,"dirs.run_cpu_time_us":378,"dirs.run_wall_time_us":6167,"drs_written":1,"lbm_read_time_us":285,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6141,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":9956,"threads_started":1}
07:09:25.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.809808 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":234,"lbm_reads_lt_1ms":4}
07:09:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.811702 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:25.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:25.813767 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":239,"lbm_reads_lt_1ms":4}
07:09:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.292007 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.314026 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:26.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.320216 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:26.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.507710 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.193s user 0.146s sys 0.015s Metrics: {"bytes_written":39776,"cfile_init":1,"dirs.queue_time_us":6965,"dirs.run_cpu_time_us":325,"dirs.run_wall_time_us":13244,"drs_written":1,"lbm_read_time_us":251,"lbm_reads_lt_1ms":4,"lbm_write_time_us":9384,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"thread_start_us":7146,"threads_started":1}
07:09:26.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.549762 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:26.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.551992 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":215,"lbm_reads_lt_1ms":4}
07:09:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.690785 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.398s user 0.200s sys 0.043s Metrics: {"bytes_written":39760,"cfile_init":1,"dirs.queue_time_us":62183,"dirs.run_cpu_time_us":315,"dirs.run_wall_time_us":1795,"drs_written":1,"lbm_read_time_us":229,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10915,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":39,"spinlock_wait_cycles":36736,"thread_start_us":62170,"threads_started":1}
07:09:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.694630 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 572 bytes on disk
07:09:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.696939 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":237,"lbm_reads_lt_1ms":4}
07:09:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.720671 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.400s user 0.177s sys 0.007s Metrics: {"bytes_written":39776,"cfile_init":1,"dirs.queue_time_us":6650,"dirs.run_cpu_time_us":302,"dirs.run_wall_time_us":24139,"drs_written":1,"lbm_read_time_us":215,"lbm_reads_lt_1ms":4,"lbm_write_time_us":29732,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"thread_start_us":6337,"threads_started":1}
07:09:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.723841 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:26.725744 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":226,"lbm_reads_lt_1ms":4}
07:09:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.057037 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.202533 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.237193 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.241497 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.183s user 0.149s sys 0.008s Metrics: {"bytes_written":39576,"cfile_init":1,"dirs.queue_time_us":9023,"dirs.run_cpu_time_us":325,"dirs.run_wall_time_us":5467,"drs_written":1,"lbm_read_time_us":312,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7323,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":29,"thread_start_us":8704,"threads_started":1}
07:09:27.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.245558 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 461 bytes on disk
07:09:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.247996 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":240,"lbm_reads_lt_1ms":4}
07:09:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.477043 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.274s user 0.164s sys 0.015s Metrics: {"bytes_written":39815,"cfile_init":1,"dirs.queue_time_us":6826,"dirs.run_cpu_time_us":318,"dirs.run_wall_time_us":12208,"drs_written":1,"lbm_read_time_us":235,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6371,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":37,"thread_start_us":7010,"threads_started":1}
07:09:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.480224 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.482337 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":266,"lbm_reads_lt_1ms":4}
07:09:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.510639 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.273s user 0.180s sys 0.008s Metrics: {"bytes_written":39809,"cfile_init":1,"dirs.queue_time_us":24204,"dirs.run_cpu_time_us":323,"dirs.run_wall_time_us":11661,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":12332,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"thread_start_us":24394,"threads_started":1}
07:09:27.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.513835 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:27.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.518590 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4}
07:09:27.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.754021 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:27.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.935405 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.181s user 0.148s sys 0.004s Metrics: {"bytes_written":39654,"cfile_init":1,"dirs.queue_time_us":915,"dirs.run_cpu_time_us":319,"dirs.run_wall_time_us":3564,"drs_written":1,"lbm_read_time_us":243,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5199,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":651,"threads_started":1}
07:09:27.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.938306 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.940331 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":232,"lbm_reads_lt_1ms":4}
07:09:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:27.987782 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.024879 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.195019 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.206s user 0.166s sys 0.008s Metrics: {"bytes_written":39781,"cfile_init":1,"dirs.queue_time_us":14655,"dirs.run_cpu_time_us":299,"dirs.run_wall_time_us":2376,"drs_written":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6304,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34,"thread_start_us":14374,"threads_started":1}
07:09:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.198505 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 516 bytes on disk
07:09:28.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.201733 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":234,"lbm_reads_lt_1ms":4}
07:09:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.243484 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.218s user 0.172s sys 0.000s Metrics: {"bytes_written":39787,"cfile_init":1,"dirs.queue_time_us":6630,"dirs.run_cpu_time_us":332,"dirs.run_wall_time_us":3769,"drs_written":1,"lbm_read_time_us":223,"lbm_reads_lt_1ms":4,"lbm_write_time_us":9913,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":35,"thread_start_us":6340,"threads_started":1}
07:09:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.247097 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 526 bytes on disk
07:09:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.249306 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":238,"lbm_reads_lt_1ms":4}
07:09:28.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.445377 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.636159 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.190s user 0.149s sys 0.008s Metrics: {"bytes_written":39693,"cfile_init":1,"dirs.queue_time_us":908,"dirs.run_cpu_time_us":386,"dirs.run_wall_time_us":1829,"drs_written":1,"lbm_read_time_us":266,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6474,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":632,"threads_started":1}
07:09:28.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.638633 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:28.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.640506 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":211,"lbm_reads_lt_1ms":4}
07:09:28.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.708477 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:28.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.756961 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.910705 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.201s user 0.157s sys 0.007s Metrics: {"bytes_written":39684,"cfile_init":1,"dirs.queue_time_us":7596,"dirs.run_cpu_time_us":287,"dirs.run_wall_time_us":1793,"drs_written":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8974,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":32,"thread_start_us":7902,"threads_started":1}
07:09:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.914186 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 492 bytes on disk
07:09:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.916484 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":255,"lbm_reads_lt_1ms":4}
07:09:28.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.977756 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.220s user 0.156s sys 0.003s Metrics: {"bytes_written":39806,"cfile_init":1,"dirs.queue_time_us":8202,"dirs.run_cpu_time_us":282,"dirs.run_wall_time_us":1766,"drs_written":1,"lbm_read_time_us":205,"lbm_reads_lt_1ms":4,"lbm_write_time_us":12827,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":33,"thread_start_us":7267,"threads_started":1}
07:09:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.980787 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:28.982764 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":238,"lbm_reads_lt_1ms":4}
07:09:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.145846 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.304025 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.157s user 0.145s sys 0.003s Metrics: {"bytes_written":39636,"cfile_init":1,"dirs.queue_time_us":5057,"dirs.run_cpu_time_us":283,"dirs.run_wall_time_us":3275,"drs_written":1,"lbm_read_time_us":200,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6163,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"thread_start_us":4818,"threads_started":1}
07:09:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.307672 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 491 bytes on disk
07:09:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.310817 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":222,"lbm_reads_lt_1ms":4}
07:09:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.422369 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.488189 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:29.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.683619 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.260s user 0.165s sys 0.019s Metrics: {"bytes_written":39795,"cfile_init":1,"dirs.queue_time_us":7981,"dirs.run_cpu_time_us":321,"dirs.run_wall_time_us":12380,"drs_written":1,"lbm_read_time_us":254,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8651,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":33,"thread_start_us":8190,"threads_started":1}
07:09:29.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.687439 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.693507 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":239,"lbm_reads_lt_1ms":4}
07:09:29.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.740870 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.250s user 0.160s sys 0.016s Metrics: {"bytes_written":39666,"cfile_init":1,"dirs.queue_time_us":10834,"dirs.run_cpu_time_us":284,"dirs.run_wall_time_us":10618,"drs_written":1,"lbm_read_time_us":241,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6807,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"spinlock_wait_cycles":518912,"thread_start_us":10622,"threads_started":1}
07:09:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.743578 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 492 bytes on disk
07:09:29.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.747246 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":167,"lbm_reads_lt_1ms":4}
07:09:29.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:29.828866 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:30.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.026518 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.197s user 0.143s sys 0.023s Metrics: {"bytes_written":39615,"cfile_init":1,"dirs.queue_time_us":5971,"dirs.run_cpu_time_us":311,"dirs.run_wall_time_us":1685,"drs_written":1,"lbm_read_time_us":215,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6171,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":29,"thread_start_us":5621,"threads_started":1}
07:09:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.030500 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 462 bytes on disk
07:09:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.032876 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.001s Metrics: {"cfile_init":1,"lbm_read_time_us":252,"lbm_reads_lt_1ms":4}
07:09:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.201134 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.253423 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.403604 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.202s user 0.172s sys 0.008s Metrics: {"bytes_written":39835,"cfile_init":1,"dirs.queue_time_us":12322,"dirs.run_cpu_time_us":311,"dirs.run_wall_time_us":8542,"drs_written":1,"lbm_read_time_us":282,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6006,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36,"spinlock_wait_cycles":60800,"thread_start_us":13984,"threads_started":1}
07:09:30.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.407107 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 536 bytes on disk
07:09:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.409564 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":238,"lbm_reads_lt_1ms":4}
07:09:30.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.538285 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.599229 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.345s user 0.171s sys 0.020s Metrics: {"bytes_written":39844,"cfile_init":1,"dirs.queue_time_us":11848,"dirs.run_cpu_time_us":303,"dirs.run_wall_time_us":9326,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11025,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":36,"thread_start_us":7814,"threads_started":1}
07:09:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.602591 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 536 bytes on disk
07:09:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.604717 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":240,"lbm_reads_lt_1ms":4}
07:09:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.728406 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.189s user 0.152s sys 0.011s Metrics: {"bytes_written":39681,"cfile_init":1,"dirs.queue_time_us":1664,"dirs.run_cpu_time_us":277,"dirs.run_wall_time_us":8955,"drs_written":1,"lbm_read_time_us":218,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5732,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"spinlock_wait_cycles":15104,"thread_start_us":1346,"threads_started":1}
07:09:30.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.731621 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.733781 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":226,"lbm_reads_lt_1ms":4}
07:09:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:30.915249 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.088943 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.173s user 0.146s sys 0.012s Metrics: {"bytes_written":39672,"cfile_init":1,"dirs.queue_time_us":6840,"dirs.run_cpu_time_us":327,"dirs.run_wall_time_us":3309,"drs_written":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6186,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"spinlock_wait_cycles":12544,"thread_start_us":6552,"threads_started":1}
07:09:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.091699 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.094110 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":229,"lbm_reads_lt_1ms":4}
07:09:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.110565 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:31.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.240270 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.311921 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.201s user 0.163s sys 0.015s Metrics: {"bytes_written":39864,"cfile_init":1,"dirs.queue_time_us":11312,"dirs.run_cpu_time_us":331,"dirs.run_wall_time_us":9194,"drs_written":1,"lbm_read_time_us":150,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7120,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"thread_start_us":11030,"threads_started":1}
07:09:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.315407 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.317941 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":282,"lbm_reads_lt_1ms":4}
07:09:31.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.469607 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.228s user 0.175s sys 0.009s Metrics: {"bytes_written":39828,"cfile_init":1,"dirs.queue_time_us":7692,"dirs.run_cpu_time_us":308,"dirs.run_wall_time_us":15124,"drs_written":1,"lbm_read_time_us":225,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5641,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34,"thread_start_us":7385,"threads_started":1}
07:09:31.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.473291 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 515 bytes on disk
07:09:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.479008 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":253,"lbm_reads_lt_1ms":4}
07:09:31.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.600436 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.776126 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.175s user 0.152s sys 0.004s Metrics: {"bytes_written":39631,"cfile_init":1,"dirs.queue_time_us":901,"dirs.run_cpu_time_us":284,"dirs.run_wall_time_us":10598,"drs_written":1,"lbm_read_time_us":215,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10765,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":29,"thread_start_us":632,"threads_started":1}
07:09:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.779132 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 461 bytes on disk
07:09:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.781579 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":254,"lbm_reads_lt_1ms":4}
07:09:31.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.824551 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:31.987717 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.050798 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.226s user 0.169s sys 0.000s Metrics: {"bytes_written":39627,"cfile_init":1,"dirs.queue_time_us":9859,"dirs.run_cpu_time_us":387,"dirs.run_wall_time_us":7106,"drs_written":1,"lbm_read_time_us":237,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6393,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":9542,"threads_started":1}
07:09:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.054332 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 481 bytes on disk
07:09:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.056859 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":290,"lbm_reads_lt_1ms":4}
07:09:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.184546 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.196s user 0.154s sys 0.012s Metrics: {"bytes_written":39589,"cfile_init":1,"dirs.queue_time_us":6845,"dirs.run_cpu_time_us":304,"dirs.run_wall_time_us":11890,"drs_written":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5899,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"thread_start_us":7047,"threads_started":1}
07:09:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.188030 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 492 bytes on disk
07:09:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.190599 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":244,"lbm_reads_lt_1ms":4}
07:09:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.287350 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.552644 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.263s user 0.173s sys 0.004s Metrics: {"bytes_written":39580,"cfile_init":1,"dirs.queue_time_us":3097,"dirs.run_cpu_time_us":401,"dirs.run_wall_time_us":1521,"drs_written":1,"lbm_read_time_us":229,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6754,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"spinlock_wait_cycles":12032,"thread_start_us":2518,"threads_started":1}
07:09:32.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.557718 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 491 bytes on disk
07:09:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.561264 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":242,"lbm_reads_lt_1ms":4}
07:09:32.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.564438 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:32.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.697613 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.745468 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.180s user 0.155s sys 0.012s Metrics: {"bytes_written":39612,"cfile_init":1,"dirs.queue_time_us":853,"dirs.run_cpu_time_us":294,"dirs.run_wall_time_us":5010,"drs_written":1,"lbm_read_time_us":226,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6929,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":536,"threads_started":1}
07:09:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.748595 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.751212 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4}
07:09:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.905218 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.207s user 0.154s sys 0.016s Metrics: {"bytes_written":39596,"cfile_init":1,"dirs.queue_time_us":7269,"dirs.run_cpu_time_us":307,"dirs.run_wall_time_us":3358,"drs_written":1,"lbm_read_time_us":234,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6716,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":6954,"threads_started":1}
07:09:32.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.908676 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:32.910766 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":200,"lbm_reads_lt_1ms":4}
07:09:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.072121 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.257553 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:33.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.291216 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.218s user 0.149s sys 0.051s Metrics: {"bytes_written":39766,"cfile_init":1,"dirs.queue_time_us":10562,"dirs.run_cpu_time_us":277,"dirs.run_wall_time_us":2988,"drs_written":1,"lbm_read_time_us":223,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6762,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":10406,"threads_started":1}
07:09:33.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.295080 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.297492 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":281,"lbm_reads_lt_1ms":4}
07:09:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.416399 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.513027 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.255s user 0.198s sys 0.023s Metrics: {"bytes_written":39588,"cfile_init":1,"dirs.queue_time_us":45390,"dirs.run_cpu_time_us":334,"dirs.run_wall_time_us":1858,"drs_written":1,"lbm_read_time_us":277,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6673,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":44988,"threads_started":1}
07:09:33.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.517565 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 481 bytes on disk
07:09:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.523130 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":281,"lbm_reads_lt_1ms":4}
07:09:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.758185 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.341s user 0.171s sys 0.016s Metrics: {"bytes_written":39642,"cfile_init":1,"dirs.queue_time_us":18183,"dirs.run_cpu_time_us":315,"dirs.run_wall_time_us":14074,"drs_written":1,"lbm_read_time_us":214,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11014,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":30,"spinlock_wait_cycles":1792,"thread_start_us":17886,"threads_started":1}
07:09:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.762451 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 471 bytes on disk
07:09:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:33.765285 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":255,"lbm_reads_lt_1ms":4}
07:09:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.056177 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.279166 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.222s user 0.159s sys 0.019s Metrics: {"bytes_written":39770,"cfile_init":1,"dirs.queue_time_us":22781,"dirs.run_cpu_time_us":457,"dirs.run_wall_time_us":1747,"drs_written":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6478,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36,"thread_start_us":22336,"threads_started":1}
07:09:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.283807 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 537 bytes on disk
07:09:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.286810 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":254,"lbm_reads_lt_1ms":4}
07:09:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.287524 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.368722 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.681905 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.311s user 0.184s sys 0.000s Metrics: {"bytes_written":39689,"cfile_init":1,"dirs.queue_time_us":10516,"dirs.run_cpu_time_us":299,"dirs.run_wall_time_us":8106,"drs_written":1,"lbm_read_time_us":222,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6576,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"spinlock_wait_cycles":188160,"thread_start_us":7219,"threads_started":1}
07:09:34.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.685508 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.693668 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":342,"lbm_reads_lt_1ms":4}
07:09:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.779582 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.491s user 0.176s sys 0.019s Metrics: {"bytes_written":39846,"cfile_init":1,"dirs.queue_time_us":32198,"dirs.run_cpu_time_us":310,"dirs.run_wall_time_us":7455,"drs_written":1,"lbm_read_time_us":259,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11926,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":35,"thread_start_us":31799,"threads_started":1}
07:09:34.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.784422 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 527 bytes on disk
07:09:34.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:34.787669 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":311,"lbm_reads_lt_1ms":4}
07:09:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.050067 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.204706 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.266788 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.216s user 0.169s sys 0.008s Metrics: {"bytes_written":39834,"cfile_init":1,"dirs.queue_time_us":721,"dirs.run_cpu_time_us":297,"dirs.run_wall_time_us":10206,"drs_written":1,"lbm_read_time_us":233,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7303,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":454,"threads_started":1}
07:09:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.270009 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:35.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.272167 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.001s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":159,"lbm_reads_lt_1ms":4}
07:09:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.295506 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.518080 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.311s user 0.172s sys 0.012s Metrics: {"bytes_written":39659,"cfile_init":1,"dirs.queue_time_us":8647,"dirs.run_cpu_time_us":327,"dirs.run_wall_time_us":12042,"drs_written":1,"lbm_read_time_us":442,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8405,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":8862,"threads_started":1}
07:09:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.523723 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.528427 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.004s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":2209,"lbm_reads_1-10_ms":1,"lbm_reads_lt_1ms":3}
07:09:35.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.595466 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.298s user 0.177s sys 0.025s Metrics: {"bytes_written":39831,"cfile_init":1,"dirs.queue_time_us":6929,"dirs.run_cpu_time_us":326,"dirs.run_wall_time_us":12057,"drs_written":1,"lbm_read_time_us":291,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6582,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36,"thread_start_us":7111,"threads_started":1}
07:09:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.599450 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 537 bytes on disk
07:09:35.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.602097 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":246,"lbm_reads_lt_1ms":4}
07:09:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.777735 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:35.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.958695 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.180s user 0.145s sys 0.008s Metrics: {"bytes_written":39641,"cfile_init":1,"dirs.queue_time_us":805,"dirs.run_cpu_time_us":262,"dirs.run_wall_time_us":3355,"drs_written":1,"lbm_read_time_us":359,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7834,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":28,"spinlock_wait_cycles":4992,"thread_start_us":512,"threads_started":1}
07:09:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.962224 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:35.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:35.964927 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":359,"lbm_reads_lt_1ms":4}
07:09:36.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.036070 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.108114 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.285559 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.249s user 0.170s sys 0.015s Metrics: {"bytes_written":39837,"cfile_init":1,"dirs.queue_time_us":19534,"dirs.run_cpu_time_us":345,"dirs.run_wall_time_us":1717,"drs_written":1,"lbm_read_time_us":240,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5907,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34,"thread_start_us":20897,"threads_started":1}
07:09:36.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.289292 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 517 bytes on disk
07:09:36.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.294379 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":240,"lbm_reads_lt_1ms":4}
07:09:36.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.321028 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.212s user 0.172s sys 0.020s Metrics: {"bytes_written":39801,"cfile_init":1,"dirs.queue_time_us":8857,"dirs.run_cpu_time_us":307,"dirs.run_wall_time_us":3484,"drs_written":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4,"lbm_write_time_us":9514,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":33,"thread_start_us":8563,"threads_started":1}
07:09:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.324738 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:36.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.331523 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.005s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4}
07:09:36.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.471132 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.655650 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.183s user 0.158s sys 0.008s Metrics: {"bytes_written":39692,"cfile_init":1,"dirs.queue_time_us":1268,"dirs.run_cpu_time_us":284,"dirs.run_wall_time_us":12619,"drs_written":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6209,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":971,"threads_started":1}
07:09:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.658471 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 471 bytes on disk
07:09:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.660930 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":222,"lbm_reads_lt_1ms":4}
07:09:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.801638 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:36.838047 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.030372 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.227s user 0.180s sys 0.000s Metrics: {"bytes_written":39673,"cfile_init":1,"dirs.queue_time_us":13729,"dirs.run_cpu_time_us":325,"dirs.run_wall_time_us":2485,"drs_written":1,"lbm_read_time_us":203,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6414,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"spinlock_wait_cycles":1792,"thread_start_us":13948,"threads_started":1}
07:09:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.033946 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 492 bytes on disk
07:09:37.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.038470 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":243,"lbm_reads_lt_1ms":4}
07:09:37.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.071350 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.233s user 0.144s sys 0.016s Metrics: {"bytes_written":39650,"cfile_init":1,"dirs.queue_time_us":6878,"dirs.run_cpu_time_us":279,"dirs.run_wall_time_us":11754,"drs_written":1,"lbm_read_time_us":206,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8401,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"spinlock_wait_cycles":1920,"thread_start_us":7049,"threads_started":1}
07:09:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.074888 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.077509 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":237,"lbm_reads_lt_1ms":4}
07:09:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.168248 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:37.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.339004 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.170s user 0.151s sys 0.008s Metrics: {"bytes_written":39647,"cfile_init":1,"dirs.queue_time_us":1213,"dirs.run_cpu_time_us":253,"dirs.run_wall_time_us":1882,"drs_written":1,"lbm_read_time_us":221,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5666,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":908,"threads_started":1}
07:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.342744 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:37.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.345836 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":222,"lbm_reads_lt_1ms":4}
07:09:37.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.545202 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:37.550179 3403 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 3338
07:09:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:37.550967 3403 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8bc0 after lost signal to thread 3406
07:09:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.585656 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.852731 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.307s user 0.175s sys 0.009s Metrics: {"bytes_written":39699,"cfile_init":1,"dirs.queue_time_us":18648,"dirs.run_cpu_time_us":288,"dirs.run_wall_time_us":5910,"drs_written":1,"lbm_read_time_us":224,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6560,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"thread_start_us":17239,"threads_started":1}
07:09:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.856554 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 491 bytes on disk
07:09:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.862717 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":270,"lbm_reads_lt_1ms":4}
07:09:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.863791 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.951130 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.364s user 0.175s sys 0.016s Metrics: {"bytes_written":39697,"cfile_init":1,"dirs.queue_time_us":8245,"dirs.run_cpu_time_us":265,"dirs.run_wall_time_us":9792,"drs_written":1,"lbm_read_time_us":226,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10773,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":8430,"threads_started":1}
07:09:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.955822 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 481 bytes on disk
07:09:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:37.959270 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":368,"lbm_reads_lt_1ms":4}
07:09:38.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.065724 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.201s user 0.152s sys 0.012s Metrics: {"bytes_written":39680,"cfile_init":1,"dirs.queue_time_us":4717,"dirs.run_cpu_time_us":306,"dirs.run_wall_time_us":4117,"drs_written":1,"lbm_read_time_us":209,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6551,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":28,"thread_start_us":4429,"threads_started":1}
07:09:38.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.069025 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.072466 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":282,"lbm_reads_lt_1ms":4}
07:09:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.369484 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.466121 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.802052 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.432s user 0.175s sys 0.011s Metrics: {"bytes_written":39687,"cfile_init":1,"dirs.queue_time_us":11185,"dirs.run_cpu_time_us":325,"dirs.run_wall_time_us":3419,"drs_written":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4,"lbm_write_time_us":17532,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":10855,"threads_started":1}
07:09:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.806020 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.812261 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.005s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":3369,"lbm_reads_1-10_ms":1,"lbm_reads_lt_1ms":3}
07:09:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.828279 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.361s user 0.213s sys 0.012s Metrics: {"bytes_written":39851,"cfile_init":1,"dirs.queue_time_us":7333,"dirs.run_cpu_time_us":539,"dirs.run_wall_time_us":14989,"drs_written":1,"lbm_read_time_us":277,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7030,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":7936,"threads_started":1}
07:09:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.832077 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.834618 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:38.834707 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4}
07:09:39.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.099184 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.264s user 0.183s sys 0.011s Metrics: {"bytes_written":39697,"cfile_init":1,"dirs.queue_time_us":4517,"dirs.run_cpu_time_us":297,"dirs.run_wall_time_us":2674,"drs_written":1,"lbm_read_time_us":258,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8117,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":31,"thread_start_us":4226,"threads_started":1}
07:09:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.102768 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.106688 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":225,"lbm_reads_lt_1ms":4}
07:09:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.323979 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.344033 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:39.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.738934 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.411s user 0.179s sys 0.055s Metrics: {"bytes_written":39661,"cfile_init":1,"dirs.queue_time_us":21712,"dirs.run_cpu_time_us":377,"dirs.run_wall_time_us":14751,"drs_written":1,"lbm_read_time_us":318,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10428,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":32,"thread_start_us":21244,"threads_started":1}
07:09:39.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.743340 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 492 bytes on disk
07:09:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.752897 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.009s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":287,"lbm_reads_lt_1ms":4}
07:09:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.874739 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.887024 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.542s user 0.191s sys 0.013s Metrics: {"bytes_written":39649,"cfile_init":1,"dirs.queue_time_us":9178,"dirs.run_cpu_time_us":345,"dirs.run_wall_time_us":8405,"drs_written":1,"lbm_read_time_us":239,"lbm_reads_lt_1ms":4,"lbm_write_time_us":12755,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":29,"thread_start_us":9428,"threads_started":1}
07:09:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.892189 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 462 bytes on disk
07:09:39.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:39.894990 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":254,"lbm_reads_lt_1ms":4}
07:09:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.139089 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.264s user 0.219s sys 0.008s Metrics: {"bytes_written":39845,"cfile_init":1,"dirs.queue_time_us":14709,"dirs.run_cpu_time_us":386,"dirs.run_wall_time_us":2963,"drs_written":1,"lbm_read_time_us":252,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6638,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":14358,"threads_started":1}
07:09:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.143177 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.145884 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":230,"lbm_reads_lt_1ms":4}
07:09:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.407083 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:40.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.512221 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:40.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.639350 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.232s user 0.179s sys 0.013s Metrics: {"bytes_written":39859,"cfile_init":1,"dirs.queue_time_us":6982,"dirs.run_cpu_time_us":353,"dirs.run_wall_time_us":13783,"drs_written":1,"lbm_read_time_us":227,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8660,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":35,"thread_start_us":7124,"threads_started":1}
07:09:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.644243 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 527 bytes on disk
07:09:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.647181 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":199,"lbm_reads_lt_1ms":4}
07:09:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.652899 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:40.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.792712 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.280s user 0.186s sys 0.024s Metrics: {"bytes_written":39869,"cfile_init":1,"dirs.queue_time_us":14201,"dirs.run_cpu_time_us":305,"dirs.run_wall_time_us":1649,"drs_written":1,"lbm_read_time_us":232,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6883,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":39,"thread_start_us":13875,"threads_started":1}
07:09:40.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.796378 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 571 bytes on disk
07:09:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.799434 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":247,"lbm_reads_lt_1ms":4}
07:09:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.837949 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.184s user 0.175s sys 0.004s Metrics: {"bytes_written":39681,"cfile_init":1,"dirs.queue_time_us":7462,"dirs.run_cpu_time_us":314,"dirs.run_wall_time_us":6272,"drs_written":1,"lbm_read_time_us":212,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6172,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":7131,"threads_started":1}
07:09:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.841547 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 470 bytes on disk
07:09:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:40.844099 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":228,"lbm_reads_lt_1ms":4}
07:09:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.154574 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.305611 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:41.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.335841 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.181s user 0.151s sys 0.016s Metrics: {"bytes_written":39855,"cfile_init":1,"dirs.queue_time_us":5723,"dirs.run_cpu_time_us":267,"dirs.run_wall_time_us":5520,"drs_written":1,"lbm_read_time_us":224,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6259,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":5379,"threads_started":1}
07:09:41.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.339397 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.341960 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":169,"lbm_reads_lt_1ms":4}
07:09:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.350339 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:41.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.643049 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.337s user 0.187s sys 0.008s Metrics: {"bytes_written":39884,"cfile_init":1,"dirs.queue_time_us":16646,"dirs.run_cpu_time_us":303,"dirs.run_wall_time_us":6666,"drs_written":1,"lbm_read_time_us":286,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6571,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":36,"thread_start_us":16864,"threads_started":1}
07:09:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.647488 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 537 bytes on disk
07:09:41.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.652415 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":346,"lbm_reads_lt_1ms":4}
07:09:41.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.689357 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.338s user 0.180s sys 0.031s Metrics: {"bytes_written":39824,"cfile_init":1,"dirs.queue_time_us":66564,"dirs.run_cpu_time_us":268,"dirs.run_wall_time_us":1615,"drs_written":1,"lbm_read_time_us":221,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6078,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"thread_start_us":65218,"threads_started":1}
07:09:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.693704 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.704902 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":265,"lbm_reads_lt_1ms":4}
07:09:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:41.857964 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.038767 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.180s user 0.140s sys 0.012s Metrics: {"bytes_written":39689,"cfile_init":1,"dirs.queue_time_us":6998,"dirs.run_cpu_time_us":364,"dirs.run_wall_time_us":1708,"drs_written":1,"lbm_read_time_us":274,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8633,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":28,"thread_start_us":6719,"threads_started":1}
07:09:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.042557 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 447 bytes on disk
07:09:42.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.045598 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":276,"lbm_reads_lt_1ms":4}
07:09:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.158552 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:42.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.211840 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.369971 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.211s user 0.181s sys 0.015s Metrics: {"bytes_written":39858,"cfile_init":1,"dirs.queue_time_us":18377,"dirs.run_cpu_time_us":342,"dirs.run_wall_time_us":1666,"drs_written":1,"lbm_read_time_us":217,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6540,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":35,"spinlock_wait_cycles":6400,"thread_start_us":18093,"threads_started":1}
07:09:42.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.374030 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 527 bytes on disk
07:09:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.378515 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":235,"lbm_reads_lt_1ms":4}
07:09:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.505935 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.293s user 0.191s sys 0.011s Metrics: {"bytes_written":39875,"cfile_init":1,"dirs.queue_time_us":13478,"dirs.run_cpu_time_us":301,"dirs.run_wall_time_us":19409,"drs_written":1,"lbm_read_time_us":233,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8158,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":37,"thread_start_us":13178,"threads_started":1}
07:09:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.510735 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 552 bytes on disk
07:09:42.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.513721 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":237,"lbm_reads_lt_1ms":4}
07:09:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.556025 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.750741 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.194s user 0.161s sys 0.004s Metrics: {"bytes_written":39668,"cfile_init":1,"dirs.queue_time_us":1392,"dirs.run_cpu_time_us":324,"dirs.run_wall_time_us":13732,"drs_written":1,"lbm_read_time_us":177,"lbm_reads_lt_1ms":4,"lbm_write_time_us":11088,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":30,"thread_start_us":1128,"threads_started":1}
07:09:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.753522 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 471 bytes on disk
07:09:42.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.756502 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":246,"lbm_reads_lt_1ms":4}
07:09:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:42.901122 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.023485 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.103686 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.202s user 0.169s sys 0.009s Metrics: {"bytes_written":39840,"cfile_init":1,"dirs.queue_time_us":2758,"dirs.run_cpu_time_us":275,"dirs.run_wall_time_us":3200,"drs_written":1,"lbm_read_time_us":299,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5710,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":33,"spinlock_wait_cycles":6404352,"thread_start_us":2455,"threads_started":1}
07:09:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.106364 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 507 bytes on disk
07:09:43.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.109609 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":268,"lbm_reads_lt_1ms":4}
07:09:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.264087 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.315033 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.291s user 0.197s sys 0.027s Metrics: {"bytes_written":39849,"cfile_init":1,"dirs.queue_time_us":7035,"dirs.run_cpu_time_us":363,"dirs.run_wall_time_us":4500,"drs_written":1,"lbm_read_time_us":241,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7716,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":35,"spinlock_wait_cycles":4352,"thread_start_us":6989,"threads_started":1}
07:09:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.319728 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 527 bytes on disk
07:09:43.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.328214 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.005s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":263,"lbm_reads_lt_1ms":4}
07:09:43.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.535019 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.270s user 0.159s sys 0.028s Metrics: {"bytes_written":39693,"cfile_init":1,"dirs.queue_time_us":2283,"dirs.run_cpu_time_us":302,"dirs.run_wall_time_us":5564,"drs_written":1,"lbm_read_time_us":232,"lbm_reads_lt_1ms":4,"lbm_write_time_us":10440,"lbm_writes_1-10_ms":2,"lbm_writes_lt_1ms":75,"peak_mem_usage":0,"rows_written":31,"thread_start_us":2015,"threads_started":1}
07:09:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.539770 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:43.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.543198 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":225,"lbm_reads_lt_1ms":4}
07:09:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.618860 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:43.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.801834 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.182s user 0.158s sys 0.011s Metrics: {"bytes_written":39642,"cfile_init":1,"dirs.queue_time_us":2287,"dirs.run_cpu_time_us":312,"dirs.run_wall_time_us":1610,"drs_written":1,"lbm_read_time_us":221,"lbm_reads_lt_1ms":4,"lbm_write_time_us":8191,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":76,"peak_mem_usage":0,"rows_written":29,"thread_start_us":2027,"threads_started":1}
07:09:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.805487 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 461 bytes on disk
07:09:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.808284 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":234,"lbm_reads_lt_1ms":4}
07:09:43.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:43.835259 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.034272 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.198s user 0.167s sys 0.013s Metrics: {"bytes_written":39824,"cfile_init":1,"dirs.queue_time_us":794,"dirs.run_cpu_time_us":296,"dirs.run_wall_time_us":1475,"drs_written":1,"lbm_read_time_us":376,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6198,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34,"thread_start_us":507,"threads_started":1}
07:09:44.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.038213 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 516 bytes on disk
07:09:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.042403 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.003s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":264,"lbm_reads_lt_1ms":4}
07:09:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.049907 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.260308 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.210s user 0.171s sys 0.008s Metrics: {"bytes_written":39856,"cfile_init":1,"dirs.queue_time_us":6799,"dirs.run_cpu_time_us":317,"dirs.run_wall_time_us":5339,"drs_written":1,"lbm_read_time_us":236,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6721,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":34,"thread_start_us":7029,"threads_started":1}
07:09:44.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.263830 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 517 bytes on disk
07:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.266240 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":178,"lbm_reads_lt_1ms":4}
07:09:44.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.315280 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.527318 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.211s user 0.184s sys 0.004s Metrics: {"bytes_written":39682,"cfile_init":1,"dirs.queue_time_us":6444,"dirs.run_cpu_time_us":284,"dirs.run_wall_time_us":4838,"drs_written":1,"lbm_read_time_us":264,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6099,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":31,"thread_start_us":6148,"threads_started":1}
07:09:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.531575 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 482 bytes on disk
07:09:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.536495 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.002s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":231,"lbm_reads_lt_1ms":4}
07:09:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.548976 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.728596 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.179s user 0.165s sys 0.008s Metrics: {"bytes_written":39647,"cfile_init":1,"dirs.queue_time_us":661,"dirs.run_cpu_time_us":304,"dirs.run_wall_time_us":2039,"drs_written":1,"lbm_read_time_us":233,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6518,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":30,"thread_start_us":392,"threads_started":1}
07:09:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.732421 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 472 bytes on disk
07:09:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.737243 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":241,"lbm_reads_lt_1ms":4}
07:09:44.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.776872 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=1.000000
07:09:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.943470 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.166s user 0.133s sys 0.016s Metrics: {"bytes_written":39612,"cfile_init":1,"dirs.queue_time_us":982,"dirs.run_cpu_time_us":292,"dirs.run_wall_time_us":1529,"drs_written":1,"lbm_read_time_us":220,"lbm_reads_lt_1ms":4,"lbm_write_time_us":6186,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":32,"spinlock_wait_cycles":13056,"thread_start_us":537,"threads_started":1}
07:09:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.946166 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 491 bytes on disk
07:09:44.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:44.948360 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":173,"lbm_reads_lt_1ms":4}
07:09:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.546937 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969091
07:09:45.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.643256 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.096s user 0.089s sys 0.004s Metrics: {"bytes_written":39591,"cfile_init":1,"dirs.queue_time_us":682,"dirs.run_cpu_time_us":220,"dirs.run_wall_time_us":1573,"drs_written":1,"lbm_read_time_us":163,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4143,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":25,"thread_start_us":424,"threads_started":1}
07:09:45.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.646147 3692 maintenance_manager.cc:419] P 322a33ac71e2422bbfa77c23587bc94f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 416 bytes on disk
07:09:45.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.648386 3575 maintenance_manager.cc:643] P 322a33ac71e2422bbfa77c23587bc94f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.001s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":184,"lbm_reads_lt_1ms":4}
07:09:45.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.746021 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.969180
07:09:45.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.829133 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.082s user 0.077s sys 0.004s Metrics: {"bytes_written":39267,"cfile_init":1,"dirs.queue_time_us":726,"dirs.run_cpu_time_us":310,"dirs.run_wall_time_us":1654,"drs_written":1,"lbm_read_time_us":165,"lbm_reads_lt_1ms":4,"lbm_write_time_us":4041,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":16,"thread_start_us":418,"threads_started":1}
07:09:45.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.832029 3831 maintenance_manager.cc:419] P 9a8646389e9548a0b1bbe4317c34a69f: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 316 bytes on disk
07:09:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.834595 3716 maintenance_manager.cc:643] P 9a8646389e9548a0b1bbe4317c34a69f: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":184,"lbm_reads_lt_1ms":4}
07:09:45.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:45.957837 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16): perf score=0.219023
07:09:46.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:46.034749 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: FlushMRSOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.076s user 0.061s sys 0.012s Metrics: {"bytes_written":39071,"cfile_init":1,"dirs.queue_time_us":743,"dirs.run_cpu_time_us":379,"dirs.run_wall_time_us":1636,"drs_written":1,"lbm_read_time_us":164,"lbm_reads_lt_1ms":4,"lbm_write_time_us":5044,"lbm_writes_lt_1ms":77,"peak_mem_usage":0,"rows_written":5,"thread_start_us":908,"threads_started":2,"wal-append.queue_time_us":766}
07:09:46.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:46.037345 3971 maintenance_manager.cc:419] P bd8f057891ad4388bf6ce94c7f9bec71: Scheduling UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16): 212 bytes on disk
07:09:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:09:46.039489 3856 maintenance_manager.cc:643] P bd8f057891ad4388bf6ce94c7f9bec71: UndoDeltaBlockGCOp(cc690e52d1e344379ce58aa35abf0e16) complete. Timing: real 0.002s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":171,"lbm_reads_lt_1ms":4}
07:09:49.887 [INFO - main] (TestKuduBackup.scala:963) nowMs: 1714115389887
07:09:49.888 [INFO - main] (TestKuduBackup.scala:965) propagated physicalMicros: 1714115384865194
07:09:49.888 [INFO - main] (TestKuduBackup.scala:966) propagated logical: 0
07:09:49.888 [INFO - main] (KuduBackup.scala:111) Backing up to root path: file:///tmp/backup1218311203741667607/
07:09:49.896 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a8de415057434381a66071c0391915f0
07:09:49.897 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:55) Backing up table split-size-table to path: file:/tmp/backup1218311203741667607/a8de415057434381a66071c0391915f0-split-size-table/1714115389887
07:09:49.897 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:66) Looking for a previous backup: forceFull and fromMs options are not set.
07:09:49.897 [INFO - ForkJoinPool-240-worker-1] (KuduBackup.scala:73) No previous backup was found. Starting a full backup.
07:09:49.898 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/tmp/dist-test-taskRfRc_3/java/build/dist-test/spark-warehouse').
07:09:49.898 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Warehouse path is 'file:/tmp/dist-test-taskRfRc_3/java/build/dist-test/spark-warehouse'.
07:09:49.916 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:49.917 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:49.917 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:49.918 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:49.918 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:49.918 [INFO - ForkJoinPool-240-worker-1] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:49.918 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:49.949 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2206) Key ranges for split-size-table
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [<start>, 0x80000014), 35128, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000014, 0x80000031), 35329, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000031, 0x8000004D), 35357, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000004D, 0x80000067), 35364, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000067, 0x80000083), 35401, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000083, 0x800000A9), 35631, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000A9, 0x800000C8), 35475, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000C8, 0x800000EF), 35580, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800000EF, 0x80000114), 35635, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000114, 0x80000136), 35601, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000136, 0x80000156), 35504, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000156, 0x80000177), 35615, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000177, 0x8000019B), 35655, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000019B, 0x800001B9), 35492, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001B9, 0x800001D6), 35451, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001D6, 0x800001F6), 35400, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800001F6, 0x80000217), 35586, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000217, 0x8000023B), 35590, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000023B, 0x8000025C), 35654, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000025C, 0x80000278), 35461, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000278, 0x80000296), 35512, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000296, 0x800002B4), 35467, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002B4, 0x800002D0), 35500, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002D0, 0x800002EF), 35517, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800002EF, 0x80000310), 35665, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000310, 0x8000032E), 35501, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000032E, 0x8000034F), 35644, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x8000034F, 0x80000374), 35695, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000374, 0x80000397), 35669, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x80000397, 0x800003B9), 35644, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800003B9, 0x800003D7), 35467, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.952 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2213) [0x800003D7, <end>), 35087, "cc690e52d1e344379ce58aa35abf0e16" [<start>, <end>)
07:09:49.970 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Starting job: save at KuduBackup.scala:97
07:09:49.971 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Got job 0 (save at KuduBackup.scala:97) with 32 output partitions
07:09:49.971 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Final stage: ResultStage 0 (save at KuduBackup.scala:97)
07:09:49.971 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Parents of final stage: List()
07:09:49.971 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Missing parents: List()
07:09:49.972 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97), which has no missing parents
07:09:49.989 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0 stored as values in memory (estimated size 189.4 KiB, free 1993.7 MiB)
07:09:49.991 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Block broadcast_0_piece0 stored as bytes in memory (estimated size 66.7 KiB, free 1993.6 MiB)
07:09:49.992 [INFO - dispatcher-BlockManagerMaster] (Logging.scala:57) Added broadcast_0_piece0 in memory on dist-test-slave-hqrx.c.gcp-upstream.internal:37193 (size: 66.7 KiB, free: 1993.8 MiB)
07:09:49.992 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Created broadcast 0 from broadcast at DAGScheduler.scala:1388
07:09:49.992 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Submitting 32 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at save at KuduBackup.scala:97) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))
07:09:49.992 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Adding task set 0.0 with 32 tasks resource profile 0
07:09:49.995 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 0.0 in stage 0.0 (TID 0) (dist-test-slave-hqrx.c.gcp-upstream.internal, executor driver, partition 0, ANY, 5273 bytes) taskResourceAssignments Map()
07:09:49.996 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Running task 0.0 in stage 0.0 (TID 0)
07:09:50.022 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:09:50.025 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a8de415057434381a66071c0391915f0
07:09:50.026 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:50.026 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (CodecConfig.java:95) Compression: SNAPPY
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:376) Parquet block size to 134217728
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:377) Parquet page size to 1048576
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:378) Parquet dictionary page size to 1048576
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:379) Dictionary is on
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:380) Validation is off
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:381) Writer version is: PARQUET_1_0
07:09:50.027 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:382) Maximum row group padding size is 8388608 bytes
07:09:50.028 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:383) Page size checking is: estimated
07:09:50.028 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:384) Min row count for page size check is: 100
07:09:50.028 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (ParquetOutputFormat.java:385) Max row count for page size check is: 10000
07:09:50.028 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Initialized Parquet WriteSupport with Catalyst schema:
{
"type" : "struct",
"fields" : [ {
"name" : "key",
"type" : "integer",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c1_i",
"type" : "integer",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c2_s",
"type" : "string",
"nullable" : true,
"metadata" : { }
}, {
"name" : "c3_double",
"type" : "double",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c4_long",
"type" : "long",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c5_bool",
"type" : "boolean",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c6_short",
"type" : "short",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c7_float",
"type" : "float",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c8_binary",
"type" : "binary",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c9_unixtime_micros",
"type" : "timestamp",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c10_byte",
"type" : "byte",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c11_decimal32",
"type" : "decimal(9,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c12_decimal64",
"type" : "decimal(18,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c13_decimal128",
"type" : "decimal(38,0)",
"nullable" : false,
"metadata" : { }
}, {
"name" : "c14_varchar",
"type" : "string",
"nullable" : true,
"metadata" : { }
}, {
"name" : "c15_date",
"type" : "date",
"nullable" : false,
"metadata" : { }
} ]
}
and corresponding Parquet message type:
message spark_schema {
required int32 key;
required int32 c1_i;
optional binary c2_s (UTF8);
required double c3_double;
required int64 c4_long;
required boolean c5_bool;
required int32 c6_short (INT_16);
required float c7_float;
required binary c8_binary;
required int96 c9_unixtime_micros;
required int32 c10_byte (INT_8);
required int32 c11_decimal32 (DECIMAL(9,0));
required int64 c12_decimal64 (DECIMAL(18,0));
required fixed_len_byte_array(16) c13_decimal128 (DECIMAL(38,0));
optional binary c14_varchar (UTF8);
required int32 c15_date (DATE);
}
07:09:50.038 [DEBUG - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:09:50.039 [DEBUG - kudu-nio-0] (Connection.java:757) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] Successfully connected to peer
07:09:50.051 [DEBUG - kudu-nio-0] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:50.054 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x12e31652, L:/127.0.0.1:56400 - R:/127.3.45.3:43935] using TOKEN/null
07:09:50.112 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 20 rows, more = false, responseScanTimestamp = 7021016636977152000) for scanner KuduScanner(table=split-size-table, tablet=cc690e52d1e344379ce58aa35abf0e16, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=0x80000014, lastPrimaryKey=<last>)
07:09:50.157 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (InternalParquetRecordWriter.java:165) Flushing mem columnStore to file. allocated memory: 37441
07:09:50.160 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (FileOutputCommitter.java:593) Saved output of task 'attempt_20240426070949918873927986881936_0000_m_000000_0' to file:/tmp/backup1218311203741667607/a8de415057434381a66071c0391915f0-split-size-table/1714115389887/_temporary/0/task_20240426070949918873927986881936_0000_m_000000
07:09:50.160 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) attempt_20240426070949918873927986881936_0000_m_000000_0: Committed
07:09:50.161 [INFO - Executor task launch worker for task 0.0 in stage 0.0 (TID 0)] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0). 6337 bytes result sent to driver
07:09:50.162 [INFO - dispatcher-event-loop-0] (Logging.scala:57) Starting task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal, executor driver, partition 1, ANY, 5279 bytes) taskResourceAssignments Map()
07:09:50.162 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Running task 1.0 in stage 0.0 (TID 1)
07:09:50.164 [INFO - task-result-getter-0] (Logging.scala:57) Finished task 0.0 in stage 0.0 (TID 0) in 170 ms on dist-test-slave-hqrx.c.gcp-upstream.internal (executor driver) (1/32)
07:09:50.170 [DEBUG - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:09:50.173 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a8de415057434381a66071c0391915f0
07:09:50.174 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:50.174 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:50.174 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:50.174 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:09:50.174 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:09:50.175 [INFO - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:09:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714115390 (unix time) try "date -d @1714115390" if you are using GNU date ***
07:09:50.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @ 0x0 (unknown)
07:09:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 3834 (TID 0x7f511b0ff700) from PID 8; stack trace: ***
07:09:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x44fd80 __tsan::CallUserSignalHandler()
07:09:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x452263 rtl_sigaction()
07:09:50.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513aa8b980 (unknown)
07:09:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f514139352b kudu::Slice::compare()
07:09:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5140734c41 kudu::operator<()
07:09:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513a1ee7c9 kudu::ScanSpec::SetLowerBoundKey()
07:09:50.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f514080454f kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
07:09:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5140804208 kudu::tablet::CFileSet::Iterator::Init()
07:09:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5140927039 kudu::tablet::DeltaApplier::Init()
07:09:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513a15ce73 kudu::MaterializingIterator::Init()
07:09:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513a159b77 kudu::InitAndMaybeWrap()
07:09:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513a15bfb6 kudu::UnionIterator::InitSubIterators()
07:09:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513a15bb13 kudu::UnionIterator::Init()
07:09:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f514070cb35 kudu::tablet::Tablet::Iterator::Init()
07:09:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5141368c24 kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
07:09:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5141366b60 kudu::tserver::TabletServiceImpl::Scan()
07:09:50.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513fa052f5 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf()::$_5::operator()()
07:09:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513fa05283 _ZNSt3__18__invokeIRZN4kudu7tserver21TabletServerServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE3$_5JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
07:09:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513fa051b2 std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513fa0512d std::__1::__function::__alloc_func<>::operator()()
07:09:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513fa043e3 std::__1::__function::__func<>::operator()()
07:09:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e133a4d std::__1::__function::__value_func<>::operator()()
07:09:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e132e87 std::__1::function<>::operator()()
07:09:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e132830 kudu::rpc::GeneratedServiceIf::Handle()
07:09:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e135c24 kudu::rpc::ServicePool::RunThread()
07:09:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e136f42 kudu::rpc::ServicePool::Init()::$_0::operator()()
07:09:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e136efa _ZNSt3__18__invokeIRZN4kudu3rpc11ServicePool4InitEiE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_
07:09:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e136e8a std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e136e52 std::__1::__function::__alloc_func<>::operator()()
07:09:50.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f513e13614e std::__1::__function::__func<>::operator()()
07:09:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x589485 std::__1::__function::__value_func<>::operator()()
07:09:50.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5892ba std::__1::function<>::operator()()
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3834)
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b34000009a8 by thread T42 (mutexes: write M334527855860383928):
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 google::(anonymous namespace)::LogFileObject::FlushUnlocked() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 (libglog.so.1+0x1acb9)
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::LogDestination::FlushLogFilesUnsafe(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:658:24 (libglog.so.1+0x1acb9)
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::FlushLogFilesUnsafe(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2064:3 (libglog.so.1+0x1acb9)
07:09:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/signalhandler.cc:353:3 (libglog.so.1+0x2ca0f)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (kudu+0x44fd7f)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::operator<(kudu::Slice const&, kudu::Slice const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/slice.h:289:12 (libtablet.so+0x359c40)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:39 (libkudu_common.so+0x20a7c8)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11 (libtablet.so+0x42954e)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3 (libtablet.so+0x429207)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3 (libtablet.so+0x54c038)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3 (libkudu_common.so+0x178e72)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 kudu::InitAndMaybeWrap(std::__1::unique_ptr<kudu::RowwiseIterator, std::__1::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3 (libkudu_common.so+0x175b76)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5 (libkudu_common.so+0x177fb5)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3 (libkudu_common.so+0x177b12)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3 (libtablet.so+0x331b34)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15 (libtserver.so+0x271c23)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16 (libtserver.so+0x26fb5f)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/tsan/src/kudu/tserver/tserver_service.service.cc:263:13 (libtserver_service_proto.so+0x3f2f4)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 decltype(std::__1::forward<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libtserver_service_proto.so+0x3f282)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_service_proto.so+0x3f1b1)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 std::__1::__function::__alloc_func<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5, std::__1::allocator<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_service_proto.so+0x3f12c)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 std::__1::__function::__func<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5, std::__1::allocator<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_service_proto.so+0x3e3e2)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f2a4c)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f1e86)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f182f)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #25 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f4c23)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #26 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f5f41)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #27 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libkrpc.so+0x1f5ef9)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f5e89)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #29 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f5e51)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #30 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f514d)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x589484)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x5892b9)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #33 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x4375f6)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7b34000009a8 by thread T1 (mutexes: write M3805):
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 google::(anonymous namespace)::LogFileObject::FlushUnlocked() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 (libglog.so.1+0x229dc)
07:09:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::(anonymous namespace)::LogFileObject::Flush() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1026:3 (libglog.so.1+0x229dc)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::AsyncLogger::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/async_logger.cc:135:17 (libkudu_util.so+0x291389)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 decltype(*(std::__1::forward<kudu::AsyncLogger*>(fp0)).*fp()) std::__1::__invoke<void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, void>(void (kudu::AsyncLogger::*&&)(), kudu::AsyncLogger*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3840:1 (libkudu_util.so+0x296aed)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (libkudu_util.so+0x2969a0)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*> >(void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (libkudu_util.so+0x2960ea)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 208 at 0x7b34000008f0 allocated by main thread:
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (kudu+0x4cf0e7)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::LogDestination::log_destination(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:920:35 (libglog.so.1+0x242f0)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::LogDestination::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:689:3 (libglog.so.1+0x242f0)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2072:3 (libglog.so.1+0x1ad4e)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:226:7 (libkudu_util.so+0x371b9a)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M334527855860383928 is already destroyed.
07:09:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3805 (0x7b34000008f8) created at:
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_rwlock_init /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1317 (kudu+0x483edb)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 glog_internal_namespace_::Mutex::Mutex() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/base/mutex.h:247:19 (libglog.so.1+0x1703a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::(anonymous namespace)::LogFileObject::LogFileObject(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:966:16 (libglog.so.1+0x1703a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::LogDestination::LogDestination(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:639:5 (libglog.so.1+0x1703a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 google::LogDestination::log_destination(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:920:39 (libglog.so.1+0x24300)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 google::LogDestination::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:689:3 (libglog.so.1+0x24300)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 google::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2072:3 (libglog.so.1+0x1ad4e)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:226:7 (libkudu_util.so+0x371b9a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T42 'rpc worker-3879' (tid=3879, running) created by main thread at:
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x4536f5)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:638:15 (libkudu_util.so+0x436e5a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.h:146:12 (kudu+0x5794d9)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::rpc::ServicePool::Init(int) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f3baf)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::RpcServer::RegisterService(std::__1::unique_ptr<kudu::rpc::ServiceIf, std::__1::default_delete<kudu::rpc::ServiceIf> >) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/server/rpc_server.cc:238:3 (libserver_process.so+0x13298f)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::server::ServerBase::RegisterService(std::__1::unique_ptr<kudu::rpc::ServiceIf, std::__1::default_delete<kudu::rpc::ServiceIf> >) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/server/server_base.cc:1171:23 (libserver_process.so+0x144d0c)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_server.cc:156:3 (libtserver.so+0x255f0a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x263123)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x607ca9)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (kudu+0x510ca1)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x510c0c)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x510b7c)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x50f7e5)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x127d2f)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x11a591)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x1182c2)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T1 (tid=3836, running) created by main thread at:
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x4536f5)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (libkudu_util.so+0x29607b)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 std::__1::thread::thread<void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, void>(void (kudu::AsyncLogger::*&&)(), kudu::AsyncLogger*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (libkudu_util.so+0x291ee7)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::AsyncLogger::Start() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/async_logger.cc:45:13 (libkudu_util.so+0x2910a2)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::(anonymous namespace)::EnableAsyncLogging() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:157:12 (libkudu_util.so+0x372268)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:289:5 (libkudu_util.so+0x371e1a)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 in google::(anonymous namespace)::LogFileObject::FlushUnlocked()
07:09:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
07:09:51.019 [DEBUG - kudu-nio-0] (Connection.java:239) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] handling channelInactive
07:09:51.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state READY due to: connection closed
07:09:51.019 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: connection closed
07:09:51.031 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:09:51.032 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] Successfully connected to peer
07:09:51.044 [DEBUG - kudu-nio-1] (SslHandler.java:1842) [id: 0xembedded, L:embedded - R:embedded] HANDSHAKEN: protocol:TLSv1.3 cipher suite:TLS_AES_128_GCM_SHA256
07:09:51.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:51.042346 3715 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.45.3:43935: connect: Connection refused (error 111)
07:09:51.047 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfb7764c2, L:/127.0.0.1:33828 - R:/127.3.45.1:35709] using TOKEN/null
07:09:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:51.048051 3715 consensus_peers.cc:485] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f -> Peer bd8f057891ad4388bf6ce94c7f9bec71 (127.3.45.3:43935): Couldn't send request to peer bd8f057891ad4388bf6ce94c7f9bec71. Status: Network error: Client connection negotiation failed: client connection to 127.3.45.3:43935: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:09:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714115391 (unix time) try "date -d @1714115391" if you are using GNU date ***
07:09:51.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @ 0x0 (unknown)
07:09:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 3531 (TID 0x7f9138cff700) from PID 8; stack trace: ***
07:09:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x44fd80 __tsan::CallUserSignalHandler()
07:09:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x452263 rtl_sigaction()
07:09:51.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915868f980 (unknown)
07:09:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915ef9752b kudu::Slice::compare()
07:09:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915e338c41 kudu::operator<()
07:09:51.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f9157df27c9 kudu::ScanSpec::SetLowerBoundKey()
07:09:51.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915e40854f kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
07:09:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915e408208 kudu::tablet::CFileSet::Iterator::Init()
07:09:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915e52b039 kudu::tablet::DeltaApplier::Init()
07:09:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f9157d60e73 kudu::MaterializingIterator::Init()
07:09:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f9157d5db77 kudu::InitAndMaybeWrap()
07:09:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f9157d5ffb6 kudu::UnionIterator::InitSubIterators()
07:09:51.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f9157d5fb13 kudu::UnionIterator::Init()
07:09:51.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915e310b35 kudu::tablet::Tablet::Iterator::Init()
07:09:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915ef6cc24 kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
07:09:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915ef6ab60 kudu::tserver::TabletServiceImpl::Scan()
07:09:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915d6092f5 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf()::$_5::operator()()
07:09:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915d609283 _ZNSt3__18__invokeIRZN4kudu7tserver21TabletServerServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE3$_5JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
07:09:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915d6091b2 std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:51.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915d60912d std::__1::__function::__alloc_func<>::operator()()
07:09:51.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915d6083e3 std::__1::__function::__func<>::operator()()
07:09:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd37a4d std::__1::__function::__value_func<>::operator()()
07:09:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd36e87 std::__1::function<>::operator()()
07:09:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd36830 kudu::rpc::GeneratedServiceIf::Handle()
07:09:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd39c24 kudu::rpc::ServicePool::RunThread()
07:09:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd3af42 kudu::rpc::ServicePool::Init()::$_0::operator()()
07:09:51.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd3aefa _ZNSt3__18__invokeIRZN4kudu3rpc11ServicePool4InitEiE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_
07:09:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd3ae8a std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:51.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd3ae52 std::__1::__function::__alloc_func<>::operator()()
07:09:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f915bd3a14e std::__1::__function::__func<>::operator()()
07:09:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x589485 std::__1::__function::__value_func<>::operator()()
07:09:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5892ba std::__1::function<>::operator()()
07:09:51.714 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] handling channelInactive
07:09:51.714 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state READY due to: connection closed
07:09:51.714 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: connection closed
07:09:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Aborted at 1714115391 (unix time) try "date -d @1714115391" if you are using GNU date ***
07:09:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) PC: @ 0x0 (unknown)
07:09:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** SIGSEGV (@0x8) received by PID 3693 (TID 0x7fb5a2df7700) from PID 8; stack trace: ***
07:09:51.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x44fd80 __tsan::CallUserSignalHandler()
07:09:51.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x452263 rtl_sigaction()
07:09:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c1dd5980 (unknown)
07:09:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c86dd52b kudu::Slice::compare()
07:09:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c7a7ec41 kudu::operator<()
07:09:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c15387c9 kudu::ScanSpec::SetLowerBoundKey()
07:09:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c7b4e54f kudu::tablet::CFileSet::Iterator::OptimizePKPredicates()
07:09:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c7b4e208 kudu::tablet::CFileSet::Iterator::Init()
07:09:51.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c7c71039 kudu::tablet::DeltaApplier::Init()
07:09:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c14a6e73 kudu::MaterializingIterator::Init()
07:09:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c14a3b77 kudu::InitAndMaybeWrap()
07:09:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c14a5fb6 kudu::UnionIterator::InitSubIterators()
07:09:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:09:51.926127 3713 consensus_peers.cc:485] T cc690e52d1e344379ce58aa35abf0e16 P 9a8646389e9548a0b1bbe4317c34a69f -> Peer 322a33ac71e2422bbfa77c23587bc94f (127.3.45.1:35709): Couldn't send request to peer 322a33ac71e2422bbfa77c23587bc94f. Status: Network error: Client connection negotiation failed: client connection to 127.3.45.1:35709: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:09:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c14a5b13 kudu::UnionIterator::Init()
07:09:51.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c7a56b35 kudu::tablet::Tablet::Iterator::Init()
07:09:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c86b2c24 kudu::tserver::TabletServiceImpl::HandleNewScanRequest()
07:09:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c86b0b60 kudu::tserver::TabletServiceImpl::Scan()
07:09:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c6d4f2f5 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf()::$_5::operator()()
07:09:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c6d4f283 _ZNSt3__18__invokeIRZN4kudu7tserver21TabletServerServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE3$_5JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_
07:09:52.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c6d4f1b2 std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c6d4f12d std::__1::__function::__alloc_func<>::operator()()
07:09:52.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c6d4e3e3 std::__1::__function::__func<>::operator()()
07:09:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c547da4d std::__1::__function::__value_func<>::operator()()
07:09:52.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c547ce87 std::__1::function<>::operator()()
07:09:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c547c830 kudu::rpc::GeneratedServiceIf::Handle()
07:09:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c547fc24 kudu::rpc::ServicePool::RunThread()
07:09:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c5480f42 kudu::rpc::ServicePool::Init()::$_0::operator()()
07:09:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c5480efa _ZNSt3__18__invokeIRZN4kudu3rpc11ServicePool4InitEiE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_
07:09:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c5480e8a std::__1::__invoke_void_return_wrapper<>::__call<>()
07:09:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c5480e52 std::__1::__function::__alloc_func<>::operator()()
07:09:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fb5c548014e std::__1::__function::__func<>::operator()()
07:09:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x589485 std::__1::__function::__value_func<>::operator()()
07:09:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5892ba std::__1::function<>::operator()()
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3693)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b34000009a8 by thread T41 (mutexes: write M146221096440908472):
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 google::(anonymous namespace)::LogFileObject::FlushUnlocked() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 (libglog.so.1+0x1acb9)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::LogDestination::FlushLogFilesUnsafe(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:658:24 (libglog.so.1+0x1acb9)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::FlushLogFilesUnsafe(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2064:3 (libglog.so.1+0x1acb9)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/signalhandler.cc:353:3 (libglog.so.1+0x2ca0f)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (kudu+0x44fd7f)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::operator<(kudu::Slice const&, kudu::Slice const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/slice.h:289:12 (libtablet.so+0x359c40)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::ScanSpec::SetLowerBoundKey(kudu::EncodedKey const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/scan_spec.cc:102:39 (libkudu_common.so+0x20a7c8)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:446:11 (libtablet.so+0x42954e)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/cfile_set.cc:410:3 (libtablet.so+0x429207)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::tablet::DeltaApplier::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/delta_applier.cc:59:3 (libtablet.so+0x54c038)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::MaterializingIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1176:3 (libkudu_common.so+0x178e72)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 kudu::InitAndMaybeWrap(std::__1::unique_ptr<kudu::RowwiseIterator, std::__1::default_delete<kudu::RowwiseIterator> >*, kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1516:3 (libkudu_common.so+0x175b76)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 kudu::UnionIterator::InitSubIterators(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1032:5 (libkudu_common.so+0x177fb5)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 kudu::UnionIterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/common/generic_iterators.cc:1005:3 (libkudu_common.so+0x177b12)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tablet::Tablet::Iterator::Init(kudu::ScanSpec*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tablet/tablet.cc:3333:3 (libtablet.so+0x331b34)
07:09:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tserver::TabletServiceImpl::HandleNewScanRequest(kudu::tablet::TabletReplica*, kudu::tserver::ScanRequestPB const*, kudu::rpc::RpcContext const*, kudu::tserver::ScanResultCollector*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >*, kudu::Timestamp*, bool*, kudu::tserver::TabletServerErrorPB_Code*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2956:15 (libtserver.so+0x271c23)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tserver::TabletServiceImpl::Scan(kudu::tserver::ScanRequestPB const*, kudu::tserver::ScanResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_service.cc:2227:16 (libtserver.so+0x26fb5f)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/tsan/src/kudu/tserver/tserver_service.service.cc:263:13 (libtserver_service_proto.so+0x3f2f4)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 decltype(std::__1::forward<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libtserver_service_proto.so+0x3f282)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_service_proto.so+0x3f1b1)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 std::__1::__function::__alloc_func<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5, std::__1::allocator<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_service_proto.so+0x3f12c)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 std::__1::__function::__func<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5, std::__1::allocator<kudu::tserver::TabletServerServiceIf::TabletServerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_5>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_service_proto.so+0x3e3e2)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f2a4c)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f1e86)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f182f)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #25 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f4c23)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #26 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f5f41)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #27 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libkrpc.so+0x1f5ef9)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f5e89)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #29 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f5e51)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #30 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f514d)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x589484)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x5892b9)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #33 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x4375f6)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7b34000009a8 by thread T1 (mutexes: write M3805):
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 google::(anonymous namespace)::LogFileObject::FlushUnlocked() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 (libglog.so.1+0x229dc)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::(anonymous namespace)::LogFileObject::Flush() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1026:3 (libglog.so.1+0x229dc)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::AsyncLogger::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/async_logger.cc:135:17 (libkudu_util.so+0x291389)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 decltype(*(std::__1::forward<kudu::AsyncLogger*>(fp0)).*fp()) std::__1::__invoke<void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, void>(void (kudu::AsyncLogger::*&&)(), kudu::AsyncLogger*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3840:1 (libkudu_util.so+0x296aed)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (libkudu_util.so+0x2969a0)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*> >(void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (libkudu_util.so+0x2960ea)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 208 at 0x7b34000008f0 allocated by main thread:
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (kudu+0x4cf0e7)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 google::LogDestination::log_destination(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:920:35 (libglog.so.1+0x242f0)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::LogDestination::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:689:3 (libglog.so.1+0x242f0)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2072:3 (libglog.so.1+0x1ad4e)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:226:7 (libkudu_util.so+0x371b9a)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M146221096440908472 is already destroyed.
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3805 (0x7b34000008f8) created at:
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_rwlock_init /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1317 (kudu+0x483edb)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 glog_internal_namespace_::Mutex::Mutex() /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/base/mutex.h:247:19 (libglog.so.1+0x1703a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 google::(anonymous namespace)::LogFileObject::LogFileObject(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:966:16 (libglog.so.1+0x1703a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 google::LogDestination::LogDestination(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:639:5 (libglog.so.1+0x1703a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 google::LogDestination::log_destination(int) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:920:39 (libglog.so.1+0x24300)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 google::LogDestination::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:689:3 (libglog.so.1+0x24300)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 google::SetLogSymlink(int, char const*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:2072:3 (libglog.so.1+0x1ad4e)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:226:7 (libkudu_util.so+0x371b9a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T41 'rpc worker-3738' (tid=3738, running) created by main thread at:
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x4536f5)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:638:15 (libkudu_util.so+0x436e5a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.h:146:12 (kudu+0x5794d9)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::rpc::ServicePool::Init(int) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f3baf)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::RpcServer::RegisterService(std::__1::unique_ptr<kudu::rpc::ServiceIf, std::__1::default_delete<kudu::rpc::ServiceIf> >) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/server/rpc_server.cc:238:3 (libserver_process.so+0x13298f)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::server::ServerBase::RegisterService(std::__1::unique_ptr<kudu::rpc::ServiceIf, std::__1::default_delete<kudu::rpc::ServiceIf> >) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/server/server_base.cc:1171:23 (libserver_process.so+0x144d0c)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_server.cc:156:3 (libtserver.so+0x255f0a)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x263123)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x607ca9)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (kudu+0x510ca1)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x510c0c)
07:09:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x510b7c)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x50f7e5)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x127d2f)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x11a591)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x1182c2)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T1 (tid=3695, running) created by main thread at:
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x4536f5)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (libkudu_util.so+0x29607b)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 std::__1::thread::thread<void (kudu::AsyncLogger::*)(), kudu::AsyncLogger*, void>(void (kudu::AsyncLogger::*&&)(), kudu::AsyncLogger*&&) /home/jenkins-slave/workspace/build_and_test@3/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (libkudu_util.so+0x291ee7)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::AsyncLogger::Start() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/async_logger.cc:45:13 (libkudu_util.so+0x2910a2)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::(anonymous namespace)::EnableAsyncLogging() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:157:12 (libkudu_util.so+0x372268)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::InitGoogleLoggingSafe(char const*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/logging.cc:289:5 (libkudu_util.so+0x371e1a)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_action.cc:242:3 (libkudu_tools_util.so+0x118264)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:138:15 (kudu+0x623a20)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:210:16 (kudu+0x624357)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 main /home/jenkins-slave/workspace/build_and_test@3/src/kudu/tools/tool_main.cc:305:10 (kudu+0x62489e)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test@3/thirdparty/src/glog-0.6.0/src/logging.cc:1037:20 in google::(anonymous namespace)::LogFileObject::FlushUnlocked()
07:09:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
07:09:52.541 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] handling channelInactive
07:09:52.542 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state READY due to: connection closed
07:09:52.542 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: connection closed
07:09:52.553 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:09:52.553 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:09:52.554 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=4, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=4, TimeoutTracker(timeout=30000, elapsed=2379), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1)))) after lookup
07:09:52.554 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:09:52.556 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.556 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.556 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.591 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:09:52.593 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:52.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:52.593 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:52.671 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:09:52.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:52.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:52.673 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:52.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:09:52.713 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:09:52.714 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=8, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=8, TimeoutTracker(timeout=30000, elapsed=2539), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240)))) after lookup
07:09:52.714 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:09:52.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.717 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:52.911 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:09:52.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:52.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:52.912 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:53.751 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:09:53.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:53.753 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:53.753 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:09:54.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:09:54.593 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:09:54.594 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=12, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=12, TimeoutTracker(timeout=30000, elapsed=4419), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525)))) after lookup
07:09:54.595 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:09:54.596 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:54.596 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:54.596 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:09:57.110 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:09:57.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:57.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:09:57.112 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:00.671 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:10:00.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:00.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:00.673 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:02.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:10:02.713 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:10:02.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=16, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=16, TimeoutTracker(timeout=30000, elapsed=12539), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449)))) after lookup
07:10:02.714 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:10:02.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:02.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:02.717 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:04.990 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:10:04.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:04.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:04.992 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:08.811 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:10:08.813 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:08.813 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:08.813 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:11.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:10:11.034 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:10:11.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=20, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=20, TimeoutTracker(timeout=30000, elapsed=20860), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271)))) after lookup
07:10:11.035 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:10:11.036 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:11.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:11.036 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:11.070 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:10:11.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:11.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:11.073 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:10:12.068046 3328 debug-util.cc:398] Leaking SignalData structure 0x7b080009c9a0 after lost signal to thread 3265
07:10:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:10:12.068620 3328 debug-util.cc:398] Leaking SignalData structure 0x7b080007db00 after lost signal to thread 3331
07:10:12.871 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:10:12.873 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:12.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:12.873 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:13.813 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:10:13.814 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:10:13.814 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=24, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=24, TimeoutTracker(timeout=30000, elapsed=23639), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153)))) after lookup
07:10:13.816 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:10:13.817 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:13.817 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:13.817 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:14.930 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:10:14.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:14.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:14.932 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:15.631 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:10:15.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:15.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:15.633 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:16.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:10:16.593 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:10:16.594 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=28, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=28, TimeoutTracker(timeout=30000, elapsed=26420), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061)))) after lookup
07:10:16.596 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:10:16.597 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:16.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:16.597 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:18.670 [DEBUG - pool-994-thread-1] (Connection.java:748) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] connecting to peer
07:10:18.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:18.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] cleaning up while in state CONNECTING due to: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:18.672 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
07:10:18.673 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
07:10:18.673 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:690) Can not open scanner
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 9 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 19 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:10:18.679 [ERROR - Executor task launch worker for task 1.0 in stage 0.0 (TID 1)] (Logging.scala:94) Exception in task 1.0 in stage 0.0 (TID 1)
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144) ~[main/:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.Task.run(Task.scala:131) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) [spark-core_2.12-3.1.2.jar:3.1.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[kudu-client-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
... 3 more
java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:10:18.729 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Starting task 2.0 in stage 0.0 (TID 2) (dist-test-slave-hqrx.c.gcp-upstream.internal, executor driver, partition 2, ANY, 5279 bytes) taskResourceAssignments Map()
07:10:18.729 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Running task 2.0 in stage 0.0 (TID 2)
07:10:18.731 [WARN - task-result-getter-1] (Logging.scala:69) Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:10:18.735 [ERROR - task-result-getter-1] (Logging.scala:73) Task 1 in stage 0.0 failed 1 times; aborting job
07:10:18.737 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Cancelling stage 0
07:10:18.737 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Killing all running tasks in stage 0: Stage cancelled
07:10:18.742 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (SecurityContext.java:320) Importing authentication credentials with one authn token, with no JWT, 1 cert(s), and realUser=slave
07:10:18.742 [INFO - dispatcher-event-loop-1] (Logging.scala:57) Executor is trying to kill task 2.0 in stage 0.0 (TID 2), reason: Stage cancelled
07:10:18.743 [INFO - dag-scheduler-event-loop] (Logging.scala:57) Stage 0 was cancelled
07:10:18.743 [INFO - dag-scheduler-event-loop] (Logging.scala:57) ResultStage 0 (save at KuduBackup.scala:97) failed in 28.770 s due to Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
07:10:18.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table a8de415057434381a66071c0391915f0
07:10:18.746 [INFO - ForkJoinPool-240-worker-1] (Logging.scala:57) Job 0 failed: save at KuduBackup.scala:97, took 28.775504 s
07:10:18.747 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:10:18.748 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:10:18.748 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:10:18.748 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:140) File Output Committer Algorithm version is 1
07:10:18.748 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (FileOutputCommitter.java:155) FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
07:10:18.748 [INFO - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Logging.scala:57) Using output committer class org.apache.parquet.hadoop.ParquetOutputCommitter
07:10:18.748 [DEBUG - Executor task launch worker for task 2.0 in stage 0.0 (TID 2)] (Connection.java:748) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] connecting to peer
07:10:18.748 [ERROR - ForkJoinPool-240-worker-1] (Logging.scala:94) Aborting job e0e7d300-ead4-4115-bdb7-80603dcb0f5b.
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
07:10:18.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:18.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] cleaning up while in state CONNECTING due to: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
at scala.Option.foreach(Option.scala:407)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
07:10:18.751 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661
07:10:18.754 [ERROR - ForkJoinPool-240-worker-1] (KuduBackup.scala:154) Failed to back up table split-size-table
org.apache.spark.SparkException: Job aborted.
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97) ~[main/:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146) ~[main/:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.Try$.apply(Try.scala:213) ~[scala-library-2.12.15.jar:?]
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146) ~[main/:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) ~[?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) ~[?:1.8.0_275]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968) ~[scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53) ~[scala-library-2.12.15.jar:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) [scala-library-2.12.15.jar:?]
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149) [scala-library-2.12.15.jar:?]
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440) [scala-library-2.12.15.jar:?]
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) [?:1.8.0_275]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_275]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_275]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_275]
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) ~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at scala.Option.foreach(Option.scala:407) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196) ~[spark-core_2.12-3.1.2.jar:3.1.2]
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200) ~[spark-sql_2.12-3.1.2.jar:3.1.2]
... 64 more
org.apache.spark.SparkException: Job aborted.
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:97)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 1 times, most recent failure: Lost task 1.0 in stage 0.0 (TID 1) (dist-test-slave-hqrx.c.gcp-upstream.internal executor driver): java.lang.RuntimeException: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:77)
at org.apache.kudu.backup.RowIterator.hasNext(KuduBackupRDD.scala:144)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:265)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:210)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=30, TimeoutTracker(timeout=30000, elapsed=28497), Traces: [0ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [844ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: connection closed, [845ms] delaying RPC due to: Network error: connection closed, [855ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [1539ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: connection closed, [1539ms] delaying RPC due to: Network error: connection closed, [1555ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2367ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: connection closed, [2367ms] delaying RPC due to: Network error: connection closed, [2375ms] refreshing cache from master, [2375ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2378ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2379ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2381ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2381ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2415ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2418ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2418ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [2498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [2536ms] refreshing cache from master, [2536ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [2538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [2539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [2542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [2735ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [2737ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [2737ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [3575ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3578ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [3578ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4415ms] refreshing cache from master, [4415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [4418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [4420ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [4421ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [4421ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [6935ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [6937ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [6937ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [10496ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [10498ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [10498ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [12535ms] refreshing cache from master, [12535ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [12538ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [12539ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [12542ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [12542ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [14815ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [14817ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [14817ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [18636ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [18638ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [18638ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [20856ms] refreshing cache from master, [20856ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [20859ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [20860ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [20861ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20861ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [20895ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [20898ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [20898ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [22696ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [22698ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [22698ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23635ms] refreshing cache from master, [23635ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [23638ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [23640ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [23642ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [23642ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [24755ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [24757ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [24757ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [25456ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [25458ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [25458ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [26415ms] refreshing cache from master, [26415ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [26418ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, [26421ms] sending RPC to server bd8f057891ad4388bf6ce94c7f9bec71, [26422ms] received response from server bd8f057891ad4388bf6ce94c7f9bec71: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [26422ms] delaying RPC due to: Network error: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935, [28495ms] sending RPC to server 322a33ac71e2422bbfa77c23587bc94f, [28497ms] received response from server 322a33ac71e2422bbfa77c23587bc94f: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, [28497ms] delaying RPC due to: Network error: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709, deferred=Deferred@1728695350(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 1.0 in stage 0.0 (TID 1) -> (continuation of Deferred@1792645439 after retry RPC@1166541240) -> (continuation of Deferred@612321016 after retry RPC@587165525) -> (continuation of Deferred@1202063191 after retry RPC@1425468449) -> (continuation of Deferred@1099341113 after retry RPC@1472561271) -> (continuation of Deferred@146980914 after retry RPC@1425035153) -> (continuation of Deferred@712188877 after retry RPC@647566061) -> (continuation of Deferred@967137646 after retry RPC@485838248))))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduScanner.nextRows(KuduScanner.java:81)
at org.apache.kudu.client.KuduScannerIterator.hasNext(KuduScannerIterator.java:69)
... 15 more
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:2260)
at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:2244)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:487)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709): Connection refused: /127.3.45.1:35709
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at org.apache.kudu.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
... 3 more
Caused by: org.apache.kudu.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.45.1:35709
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at org.apache.kudu.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at org.apache.kudu.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at org.apache.kudu.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at org.apache.kudu.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at org.apache.kudu.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2258)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2207)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2206)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2206)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1079)
at scala.Option.foreach(Option.scala:407)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1079)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2445)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2387)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2376)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
... 64 more
07:10:18.759 [ERROR - ForkJoinPool-240-worker-1] (KuduBackup.scala:162) Failed to back up table split-size-table: Look back in the logs for the full exception. Error: Failure(org.apache.spark.SparkException: Job aborted.)
07:10:18.764 [INFO - dispatcher-event-loop-1] (Logging.scala:57) MapOutputTrackerMasterEndpoint stopped!
07:10:18.767 [INFO - main] (Logging.scala:57) MemoryStore cleared
07:10:18.768 [INFO - main] (Logging.scala:57) BlockManager stopped
07:10:18.768 [INFO - main] (Logging.scala:57) BlockManagerMaster stopped
07:10:18.768 [INFO - dispatcher-event-loop-0] (Logging.scala:57) OutputCommitCoordinator stopped!
07:10:18.773 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet cc690e52d1e344379ce58aa35abf0e16 for table 'split-size-table' with partition [<start>, <end>)
07:10:18.773 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=cc690e52d1e344379ce58aa35abf0e16@[322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709),9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)[L],bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)]}]
07:10:18.774 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC ScanRequest(scannerId=null, state=OPENING, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=2, KuduRpc(method=Scan, tablet=cc690e52d1e344379ce58aa35abf0e16, attempt=2, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 9a8646389e9548a0b1bbe4317c34a69f, [3ms] received response from server 9a8646389e9548a0b1bbe4317c34a69f: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [4ms] delaying RPC due to: Network error: Failed to connect to peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661): Connection refused: /127.3.45.2:44661, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.45.62:33155, [25ms] Sub RPC GetTableLocations: received response from server master-127.3.45.62:33155: OK, deferred=Deferred@1419431449(state=PENDING, result=null, callback=scanner opened -> passthrough -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2), errback=passthrough -> open scanner errback -> wakeup thread Executor task launch worker for task 2.0 in stage 0.0 (TID 2)))) after lookup
07:10:18.774 [DEBUG - kudu-nio-0] (Connection.java:748) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] connecting to peer
07:10:18.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:18.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] cleaning up while in state CONNECTING due to: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:18.777 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935) for tablet cc690e52d1e344379ce58aa35abf0e16: Failed to connect to peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935): Connection refused: /127.3.45.3:43935
07:10:18.779 [INFO - main] (Logging.scala:57) Successfully stopped SparkContext
07:10:18.779 [DEBUG - main] (Connection.java:575) [peer bd8f057891ad4388bf6ce94c7f9bec71(127.3.45.3:43935)] disconnecting while in state TERMINATED
07:10:18.779 [DEBUG - main] (Connection.java:575) [peer master-127.3.45.62:33155(127.3.45.62:33155)] disconnecting while in state READY
07:10:18.779 [DEBUG - main] (Connection.java:575) [peer 9a8646389e9548a0b1bbe4317c34a69f(127.3.45.2:44661)] disconnecting while in state TERMINATED
07:10:18.779 [DEBUG - main] (Connection.java:575) [peer master-127.3.45.61:42905(127.3.45.61:42905)] disconnecting while in state READY
07:10:18.780 [DEBUG - main] (Connection.java:575) [peer master-127.3.45.60:44303(127.3.45.60:44303)] disconnecting while in state READY
07:10:18.780 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.45.61:42905(127.3.45.61:42905)] handling channelInactive
07:10:18.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.45.61:42905(127.3.45.61:42905)] cleaning up while in state READY due to: connection closed
07:10:18.780 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.45.62:33155(127.3.45.62:33155)] handling channelInactive
07:10:18.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.45.62:33155(127.3.45.62:33155)] cleaning up while in state READY due to: connection closed
07:10:18.780 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.45.60:44303(127.3.45.60:44303)] handling channelInactive
07:10:18.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.45.60:44303(127.3.45.60:44303)] cleaning up while in state READY due to: connection closed
07:10:18.781 [DEBUG - main] (Connection.java:575) [peer 322a33ac71e2422bbfa77c23587bc94f(127.3.45.1:35709)] disconnecting while in state TERMINATED
07:10:18.781 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:10:18.782 [DEBUG - kudu-nio-1] (PoolThreadCache.java:221) Freed 31 thread-local buffer(s) from thread: kudu-nio-1
07:10:18.783 [DEBUG - kudu-nio-0] (PoolThreadCache.java:221) Freed 37 thread-local buffer(s) from thread: kudu-nio-0
07:10:18.792 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:10:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:10:18.794276 3252 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu with pid 3264
07:10:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:10:18.830533 3252 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu with pid 3334
07:10:18.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:10:18.879258 3252 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRfRc_3/build/tsan/bin/kudu with pid 3409
07:10:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:10:19Z chronyd exiting
07:10:21.014 [ERROR - main] (RetryRule.java:206) org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes: failed attempt 1
java.lang.AssertionError: null
at org.junit.Assert.fail(Assert.java:87) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:53) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414) ~[test/:?]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40) ~[kudu-spark3_2.12-1.18.0-SNAPSHOT-tests.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.AssertionError
at org.junit.Assert.fail(Assert.java:87)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.junit.Assert.assertTrue(Assert.java:53)
at org.apache.kudu.backup.TestKuduBackup.$anonfun$testBackupWithSplitSizeBytes$1(TestKuduBackup.scala:414)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withListener(SparkListenerUtil.scala:66)
at org.apache.kudu.spark.kudu.SparkListenerUtil$.withJobTaskCounter(SparkListenerUtil.scala:40)
at org.apache.kudu.backup.TestKuduBackup.testBackupWithSplitSizeBytes(TestKuduBackup.scala:413)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)