02:02:39.843 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:02:39.855 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXxtDsG/test-tmp
02:02:39.858 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:02:39.861 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXxtDsG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:02:40.102 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:02:40.235 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:02:40.236 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:02:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.240396 20398 mini_oidc.cc:152] Starting JWKS server
02:02:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.244141 20398 webserver.cc:460] Webserver started at http://127.0.0.1:41857/ using document root <none> and password file <none>
02:02:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.244700 20398 mini_oidc.cc:189] Starting OIDC Discovery server
02:02:40.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.246920 20398 webserver.cc:460] Webserver started at http://127.0.0.1:35215/ using document root <none> and password file <none>
02:02:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:02:40Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:02:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:02:40Z Disabled control of system clock
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.307538 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/logs
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data/info.pb
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.190:36235
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.190
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json with env {}
02:02:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.597555 20410 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.598294 20410 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.599021 20410 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.629344 20410 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.629747 20410 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.630067 20410 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.630393 20410 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.630730 20410 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.631003 20410 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
02:02:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.665618 20410 master_runner.cc:386] Master server non-default flags:
02:02:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:40.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.190:36235
02:02:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data/info.pb
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.190
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/logs
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.667241 20410 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.669564 20410 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.685330 20416 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.685320 20417 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:40.687306 20410 server_base.cc:1034] running on GCE node
02:02:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:40.685863 20419 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:41.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.719518 20410 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.722697 20410 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:41.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.724152 20410 hybrid_clock.cc:648] HybridClock initialized: now 1714615361724109 us; error 64 us; skew 500 ppm
02:02:41.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.724978 20410 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.737365 20410 webserver.cc:460] Webserver started at http://127.19.235.190:38333/ using document root <none> and password file <none>
02:02:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.738346 20410 fs_manager.cc:359] Metadata directory not provided
02:02:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.738551 20410 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:41.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.738955 20410 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.743927 20410 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data/instance:
02:02:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cea01313a13c45d4b7e2a476631b60e9"
02:02:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:41 on dist-test-slave-66v3"
02:02:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.745146 20410 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal/instance:
02:02:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cea01313a13c45d4b7e2a476631b60e9"
02:02:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:41 on dist-test-slave-66v3"
02:02:41.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.752501 20410 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
02:02:41.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.758579 20426 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:41.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.759743 20410 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:02:41.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.760120 20410 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cea01313a13c45d4b7e2a476631b60e9"
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:41 on dist-test-slave-66v3"
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.760546 20410 fs_report.cc:383] FS layout report
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal
02:02:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data/data
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:41.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:41.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:41.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:41.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.807843 20410 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:41.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.809478 20410 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.810024 20410 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.878074 20410 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.190:36235
02:02:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.878183 20477 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.190:36235 every 8 connection(s)
02:02:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.885087 20410 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/data/info.pb
02:02:41.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.887913 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20410
02:02:41.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.888456 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-0/wal/instance
02:02:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.891763 20478 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.893537 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/logs
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data/info.pb
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.189:35125
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.189
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:41.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json with env {}
02:02:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.908841 20478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } has no permanent_uuid. Determining permanent_uuid...
02:02:41.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.936296 20478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } has no permanent_uuid. Determining permanent_uuid...
02:02:41.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:41.938899 20429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:41.942850 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:41.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:41.992491 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 1
02:02:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:41.996573 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.065003 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 2
02:02:42.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.069407 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.160787 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 3
02:02:42.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.164978 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.203589 20480 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.204308 20480 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:42.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.204993 20480 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.235385 20480 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.235743 20480 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.236068 20480 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.236404 20480 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.236770 20480 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.237008 20480 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.271687 20480 master_runner.cc:386] Master server non-default flags:
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data
02:02:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.189:35125
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data/info.pb
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.189
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/logs
02:02:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:42.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:42.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:42.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:42.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.273319 20480 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.275679 20480 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:42.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.291718 20489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:42.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.308423 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 4
02:02:42.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.312517 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:42.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.612066 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 5
02:02:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.616305 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.163843 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 6
02:02:43.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.170570 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.189:35125: Network error: Client connection negotiation failed: client connection to 127.19.235.189:35125: connect: Connection refused (error 111)
02:02:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.292024 20492 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:42.292338 20490 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:42.293252 20480 server_base.cc:1034] running on GCE node
02:02:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.334172 20480 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:43.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.336992 20480 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:43.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.338460 20480 hybrid_clock.cc:648] HybridClock initialized: now 1714615363338408 us; error 80 us; skew 500 ppm
02:02:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.339675 20480 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.346263 20480 webserver.cc:460] Webserver started at http://127.19.235.189:42999/ using document root <none> and password file <none>
02:02:43.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.347219 20480 fs_manager.cc:359] Metadata directory not provided
02:02:43.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.347427 20480 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:43.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.347811 20480 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:43.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.352845 20480 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data/instance:
02:02:43.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b25912c882347b4bf8d467cb4e4228f"
02:02:43.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:43 on dist-test-slave-66v3"
02:02:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.354056 20480 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal/instance:
02:02:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b25912c882347b4bf8d467cb4e4228f"
02:02:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:43 on dist-test-slave-66v3"
02:02:43.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.361536 20480 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.004s
02:02:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.367338 20501 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:43.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.368590 20480 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.368933 20480 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b25912c882347b4bf8d467cb4e4228f"
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:43 on dist-test-slave-66v3"
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.369307 20480 fs_report.cc:383] FS layout report
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal
02:02:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal
02:02:43.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data/data
02:02:43.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:43.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:43.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:43.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.433532 20480 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.435163 20480 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.435622 20480 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.505849 20480 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.189:35125
02:02:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.505928 20552 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.189:35125 every 8 connection(s)
02:02:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.508862 20480 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/data/info.pb
02:02:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.514587 20553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:43.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.518183 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20480
02:02:43.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.518613 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-1/wal/instance
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.520781 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/logs
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data/info.pb
02:02:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.188:42729
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.188
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json with env {}
02:02:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.527011 20553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } has no permanent_uuid. Determining permanent_uuid...
02:02:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.553263 20553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } has no permanent_uuid. Determining permanent_uuid...
02:02:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.566915 20553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } has no permanent_uuid. Determining permanent_uuid...
02:02:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.569342 20505 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.573407 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.622977 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 1
02:02:43.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.626979 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:43.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.695389 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 2
02:02:43.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.699540 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.790966 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 3
02:02:43.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.795699 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:43.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.826391 20555 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:43.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.826939 20555 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:43.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.827443 20555 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.858034 20555 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.858332 20555 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.858544 20555 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.858793 20555 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.859001 20555 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.859149 20555 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41857/jwks.json
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.893983 20555 master_runner.cc:386] Master server non-default flags:
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.188:42729
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41857/jwks.json
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data/info.pb
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.188
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/logs
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.895149 20555 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:43.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.897346 20555 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:43.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.915763 20565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:43.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.939182 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 4
02:02:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.943782 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.231108 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } attempt: 7
02:02:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.242640 20478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } has no permanent_uuid. Determining permanent_uuid...
02:02:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.243171 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 5
02:02:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.248709 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.249564 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.308023 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 1
02:02:44.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.312539 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.393924 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 2
02:02:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.398214 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.483584 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 3
02:02:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.487709 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.628134 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 4
02:02:44.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.632572 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.796157 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 6
02:02:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.802523 20553 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.915977 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 5
02:02:44.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:44.919853 20478 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.235.188:42729: Network error: Client connection negotiation failed: client connection to 127.19.235.188:42729: connect: Connection refused (error 111)
02:02:44.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.916147 20566 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:44.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:43.916538 20568 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:44.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:43.917773 20555 server_base.cc:1034] running on GCE node
02:02:44.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.957758 20555 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.960382 20555 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.961771 20555 hybrid_clock.cc:648] HybridClock initialized: now 1714615364961734 us; error 48 us; skew 500 ppm
02:02:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.962707 20555 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:44.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.969406 20555 webserver.cc:460] Webserver started at http://127.19.235.188:37709/ using document root <none> and password file <none>
02:02:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.970521 20555 fs_manager.cc:359] Metadata directory not provided
02:02:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.970796 20555 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.971388 20555 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.976814 20555 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data/instance:
02:02:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9a39f7a7e044a72bb8b9f7432897905"
02:02:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:44 on dist-test-slave-66v3"
02:02:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.978209 20555 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal/instance:
02:02:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9a39f7a7e044a72bb8b9f7432897905"
02:02:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:44 on dist-test-slave-66v3"
02:02:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.986289 20555 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
02:02:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.992089 20580 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.993255 20555 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.993633 20555 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9a39f7a7e044a72bb8b9f7432897905"
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:44 on dist-test-slave-66v3"
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:44.994072 20555 fs_report.cc:383] FS layout report
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal
02:02:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data/data
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:45.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.038169 20555 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.039765 20555 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.040273 20555 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.106552 20555 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.188:42729
02:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.106598 20631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.188:42729 every 8 connection(s)
02:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.109298 20555 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/data/info.pb
02:02:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.114367 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20555
02:02:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.114725 20632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.114805 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/master-2/wal/instance
02:02:45.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.131916 20632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } has no permanent_uuid. Determining permanent_uuid...
02:02:45.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.157526 20632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } has no permanent_uuid. Determining permanent_uuid...
02:02:45.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.167359 20632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } has no permanent_uuid. Determining permanent_uuid...
02:02:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.182888 20632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: Bootstrap starting.
02:02:45.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.188390 20632 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: Neither blocks nor log segments found. Creating new log.
02:02:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.190640 20632 log.cc:824] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: Log is configured to *not* fsync() on all Append() calls
02:02:45.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.194988 20632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: No bootstrap required, opened a new log
02:02:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.215703 20632 raft_consensus.cc:348] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.216589 20632 raft_consensus.cc:374] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:02:45.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.216878 20632 raft_consensus.cc:729] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9a39f7a7e044a72bb8b9f7432897905, State: Initialized, Role: FOLLOWER
02:02:45.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.217823 20632 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.221176 20641 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.222232 20641 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: This master's current role is: FOLLOWER
02:02:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.222410 20632 sys_catalog.cc:564] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: configured and running, proceeding with master startup.
02:02:45.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.246623 20652 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:02:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.249262 20652 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:02:45.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.242378 20555 master_runner.cc:186] Error getting master registration for 127.19.235.190:36235: OK, instance_id { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" instance_seqno: 1714615361858873 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:02:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.265830 20555 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:02:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.295301 20586 tablet.cc:2369] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:02:45.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.417162 20641 raft_consensus.cc:482] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:02:45.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.417726 20641 raft_consensus.cc:504] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.421272 20641 leader_election.cc:288] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cea01313a13c45d4b7e2a476631b60e9 (127.19.235.190:36235), 2b25912c882347b4bf8d467cb4e4228f (127.19.235.189:35125)
02:02:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.472194 20478 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 6
02:02:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.493054 20478 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: Bootstrap starting.
02:02:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.498472 20478 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: Neither blocks nor log segments found. Creating new log.
02:02:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.500766 20478 log.cc:824] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: Log is configured to *not* fsync() on all Append() calls
02:02:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.511816 20478 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: No bootstrap required, opened a new log
02:02:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.553396 20478 raft_consensus.cc:348] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.554792 20478 raft_consensus.cc:374] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:02:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.555954 20478 raft_consensus.cc:729] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cea01313a13c45d4b7e2a476631b60e9, State: Initialized, Role: FOLLOWER
02:02:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.557161 20478 consensus_queue.cc:260] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.561074 20656 sys_catalog.cc:455] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.563020 20656 sys_catalog.cc:458] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: This master's current role is: FOLLOWER
02:02:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.564955 20478 sys_catalog.cc:564] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: configured and running, proceeding with master startup.
02:02:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.565214 20453 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9a39f7a7e044a72bb8b9f7432897905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cea01313a13c45d4b7e2a476631b60e9" is_pre_election: true
02:02:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.566886 20453 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d9a39f7a7e044a72bb8b9f7432897905 in term 0.
02:02:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.568897 20584 leader_election.cc:302] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cea01313a13c45d4b7e2a476631b60e9, d9a39f7a7e044a72bb8b9f7432897905; no voters:
02:02:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.569921 20641 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:02:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.570339 20641 raft_consensus.cc:482] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:02:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.570493 20432 tablet.cc:2369] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:02:45.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.570634 20641 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 0 FOLLOWER]: Advancing to term 1
02:02:45.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.579106 20641 raft_consensus.cc:504] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.589546 20453 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9a39f7a7e044a72bb8b9f7432897905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cea01313a13c45d4b7e2a476631b60e9"
02:02:45.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.590209 20453 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 0 FOLLOWER]: Advancing to term 1
02:02:45.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.597184 20641 leader_election.cc:288] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [CANDIDATE]: Term 1 election: Requested vote from peers cea01313a13c45d4b7e2a476631b60e9 (127.19.235.190:36235), 2b25912c882347b4bf8d467cb4e4228f (127.19.235.189:35125)
02:02:45.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.600162 20453 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d9a39f7a7e044a72bb8b9f7432897905 in term 1.
02:02:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.601438 20584 leader_election.cc:302] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cea01313a13c45d4b7e2a476631b60e9, d9a39f7a7e044a72bb8b9f7432897905; no voters:
02:02:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.602815 20641 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 1 FOLLOWER]: Leader election won for term 1
02:02:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.609007 20641 raft_consensus.cc:686] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [term 1 LEADER]: Becoming Leader. State: Replica: d9a39f7a7e044a72bb8b9f7432897905, State: Running, Role: LEADER
02:02:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.610036 20641 consensus_queue.cc:237] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.624503 20663 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9a39f7a7e044a72bb8b9f7432897905. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.625391 20663 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: This master's current role is: LEADER
02:02:45.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.631136 20666 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:02:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.639029 20666 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:02:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.668186 20671 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:02:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:45.668946 20671 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:02:45.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.674448 20453 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [term 1 FOLLOWER]: Refusing update from remote peer d9a39f7a7e044a72bb8b9f7432897905: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:02:45.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.676319 20641 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:02:45.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.666504 20410 master_runner.cc:186] Error getting master registration for 127.19.235.189:35125: OK, instance_id { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" instance_seqno: 1714615363485756 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:02:45.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.749939 20410 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:02:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.767730 20668 mvcc.cc:205] Tried to move back new op lower bound from 7023064537781452800 to 7023064537552232448. Current Snapshot: MvccSnapshot[applied={T|T < 7023064537781452800}]
02:02:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.770869 20656 sys_catalog.cc:455] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9a39f7a7e044a72bb8b9f7432897905. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.773872 20656 sys_catalog.cc:458] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: This master's current role is: FOLLOWER
02:02:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.783573 20641 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.784360 20641 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: This master's current role is: LEADER
02:02:45.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.789616 20666 catalog_manager.cc:1347] Generated new cluster ID: 44f838f579cf4e51854f23dc4af1a201
02:02:45.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.789968 20666 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:02:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.798190 20656 sys_catalog.cc:455] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.798934 20656 sys_catalog.cc:458] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9 [sys.catalog]: This master's current role is: FOLLOWER
02:02:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.839500 20666 catalog_manager.cc:1370] Generated new certificate authority record
02:02:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.842044 20666 catalog_manager.cc:1504] Loading token signing keys...
02:02:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.862828 20553 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } attempt: 7
02:02:45.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.877090 20666 catalog_manager.cc:5857] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905: Generated new TSK 0
02:02:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.878211 20666 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:02:45.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.900908 20553 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: Bootstrap starting.
02:02:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.918190 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal
02:02:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/logs
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data/info.pb
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.129:0
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.129
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.129
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:02:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.924516 20553 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: Neither blocks nor log segments found. Creating new log.
02:02:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.927763 20553 log.cc:824] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: Log is configured to *not* fsync() on all Append() calls
02:02:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.939044 20553 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: No bootstrap required, opened a new log
02:02:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.971480 20553 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.972741 20553 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:02:45.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.973263 20553 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2b25912c882347b4bf8d467cb4e4228f, State: Initialized, Role: FOLLOWER
02:02:45.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.974470 20553 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } }
02:02:45.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.983067 20681 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.988418 20553 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: configured and running, proceeding with master startup.
02:02:45.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.989348 20527 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9a39f7a7e044a72bb8b9f7432897905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b25912c882347b4bf8d467cb4e4228f"
02:02:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.988600 20528 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9a39f7a7e044a72bb8b9f7432897905" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b25912c882347b4bf8d467cb4e4228f" is_pre_election: true
02:02:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.989025 20526 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [term 0 FOLLOWER]: Advancing to term 1
02:02:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.988992 20681 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: This master's current role is: FOLLOWER
02:02:45.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:45.998888 20526 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [term 1 FOLLOWER]: Refusing update from remote peer d9a39f7a7e044a72bb8b9f7432897905: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:02:46.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.001089 20677 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:02:46.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.094599 20691 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.097146 20691 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: This master's current role is: FOLLOWER
02:02:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.096349 20681 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9a39f7a7e044a72bb8b9f7432897905. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.099084 20681 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f [sys.catalog]: This master's current role is: FOLLOWER
02:02:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.099200 20697 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:02:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.474400 20677 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9a39f7a7e044a72bb8b9f7432897905" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cea01313a13c45d4b7e2a476631b60e9" member_type: VOTER last_known_addr { host: "127.19.235.190" port: 36235 } } peers { permanent_uuid: "2b25912c882347b4bf8d467cb4e4228f" member_type: VOTER last_known_addr { host: "127.19.235.189" port: 35125 } } peers { permanent_uuid: "d9a39f7a7e044a72bb8b9f7432897905" member_type: VOTER last_known_addr { host: "127.19.235.188" port: 42729 } } }
02:02:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.475224 20677 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9a39f7a7e044a72bb8b9f7432897905 [sys.catalog]: This master's current role is: LEADER
02:02:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.552872 20679 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.553579 20679 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.554394 20679 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:46.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.610865 20679 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:46.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.612383 20679 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.235.129
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.673875 20679 tablet_server_runner.cc:78] Tablet server non-default flags:
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.129:0
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data/info.pb
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.129
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.129
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/logs
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.675508 20679 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.677712 20679 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.678018 20671 catalog_manager.cc:1259] Loaded cluster ID: 44f838f579cf4e51854f23dc4af1a201
02:02:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.678287 20671 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: loading cluster ID for follower catalog manager: success
02:02:46.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.686595 20671 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: acquiring CA information for follower catalog manager: success
02:02:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:46.696126 20671 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cea01313a13c45d4b7e2a476631b60e9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:02:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.711807 20705 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:47.131449 20697 catalog_manager.cc:1259] Loaded cluster ID: 44f838f579cf4e51854f23dc4af1a201
02:02:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:47.131832 20697 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: loading cluster ID for follower catalog manager: success
02:02:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:47.157809 20697 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: acquiring CA information for follower catalog manager: success
02:02:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:47.173523 20697 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2b25912c882347b4bf8d467cb4e4228f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:02:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:48.104209 20704 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 20679
02:02:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:46.712512 20706 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:48.337788 20704 kernel_stack_watchdog.cc:198] Thread 20679 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 393ms:
02:02:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:02:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:02:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:02:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:02:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:48.344084 20679 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.636s user 0.000s sys 0.008s
02:02:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:48.344488 20679 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.637s user 0.000s sys 0.008s
02:02:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.355201 20679 server_base.cc:1034] running on GCE node
02:02:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:48.355792 20710 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.357180 20679 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.360052 20679 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:48.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.361620 20679 hybrid_clock.cc:648] HybridClock initialized: now 1714615368361470 us; error 160 us; skew 500 ppm
02:02:48.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.362877 20679 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.377120 20679 webserver.cc:460] Webserver started at http://127.19.235.129:38653/ using document root <none> and password file <none>
02:02:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.378429 20679 fs_manager.cc:359] Metadata directory not provided
02:02:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.378720 20679 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.379295 20679 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.386819 20679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data/instance:
02:02:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f04d0cae7947de83ecc99b3ecd5bb6"
02:02:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:48 on dist-test-slave-66v3"
02:02:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.388386 20679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal/instance:
02:02:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f04d0cae7947de83ecc99b3ecd5bb6"
02:02:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:48 on dist-test-slave-66v3"
02:02:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.399399 20679 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.008s sys 0.000s
02:02:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.407925 20716 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:48.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.409380 20679 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
02:02:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.409812 20679 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal
02:02:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f04d0cae7947de83ecc99b3ecd5bb6"
02:02:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:48 on dist-test-slave-66v3"
02:02:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.410251 20679 fs_report.cc:383] FS layout report
02:02:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data/data
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.502775 20679 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.504868 20679 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.505439 20679 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:48.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.524152 20679 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:02:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.538142 20679 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:02:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.538453 20679 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:02:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.538743 20679 ts_tablet_manager.cc:590] Registered 0 tablets
02:02:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.538941 20679 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:02:48.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.747756 20679 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.129:45301
02:02:48.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.747809 20828 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.129:45301 every 8 connection(s)
02:02:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.751538 20679 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/data/info.pb
02:02:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.761713 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20679
02:02:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.762271 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-0/wal/instance
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.774564 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/logs
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data/info.pb
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.130:0
02:02:48.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.130
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.130
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:02:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.821282 20829 heartbeater.cc:347] Connected to a master server at 127.19.235.188:42729
02:02:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.821830 20829 heartbeater.cc:464] Registering TS with master...
02:02:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.823148 20829 heartbeater.cc:511] Master 127.19.235.188:42729 requested a full tablet report, sending...
02:02:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.826409 20830 heartbeater.cc:347] Connected to a master server at 127.19.235.189:35125
02:02:48.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.826762 20830 heartbeater.cc:464] Registering TS with master...
02:02:48.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.827275 20831 heartbeater.cc:347] Connected to a master server at 127.19.235.190:36235
02:02:48.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.827623 20831 heartbeater.cc:464] Registering TS with master...
02:02:48.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.827904 20830 heartbeater.cc:511] Master 127.19.235.189:35125 requested a full tablet report, sending...
02:02:48.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.828490 20831 heartbeater.cc:511] Master 127.19.235.190:36235 requested a full tablet report, sending...
02:02:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.831918 20518 ts_manager.cc:194] Registered new tserver with Master: 25f04d0cae7947de83ecc99b3ecd5bb6 (127.19.235.129:45301)
02:02:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.831967 20443 ts_manager.cc:194] Registered new tserver with Master: 25f04d0cae7947de83ecc99b3ecd5bb6 (127.19.235.129:45301)
02:02:48.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.834750 20597 ts_manager.cc:194] Registered new tserver with Master: 25f04d0cae7947de83ecc99b3ecd5bb6 (127.19.235.129:45301)
02:02:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:48.837404 20597 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.235.129:59681
02:02:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.138198 20838 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.138732 20838 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.139312 20838 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:49.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.173260 20838 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.174379 20838 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.235.130
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:49.209756 20838 tablet_server_runner.cc:78] Tablet server non-default flags:
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.130:0
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data/info.pb
02:02:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.130
02:02:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.130
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/logs
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:49.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:49.211385 20838 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:49.213550 20838 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:49.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.234679 20846 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:49.841368 20829 heartbeater.cc:503] Master 127.19.235.188:42729 was elected leader, sending a full tablet report...
02:02:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.235407 20845 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:49.237884 20848 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.298357 20847 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:02:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.298434 20838 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:02:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.302107 20838 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.304394 20838 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.305757 20838 hybrid_clock.cc:648] HybridClock initialized: now 1714615370305723 us; error 49 us; skew 500 ppm
02:02:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.306591 20838 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.313673 20838 webserver.cc:460] Webserver started at http://127.19.235.130:36841/ using document root <none> and password file <none>
02:02:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.314625 20838 fs_manager.cc:359] Metadata directory not provided
02:02:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.314826 20838 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.315253 20838 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.320574 20838 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data/instance:
02:02:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2060b26df6646bf962abad067f721ef"
02:02:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:50 on dist-test-slave-66v3"
02:02:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.321720 20838 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal/instance:
02:02:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2060b26df6646bf962abad067f721ef"
02:02:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:50 on dist-test-slave-66v3"
02:02:50.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.329893 20838 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.005s
02:02:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.335654 20856 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.337304 20838 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.337603 20838 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2060b26df6646bf962abad067f721ef"
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:50 on dist-test-slave-66v3"
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.338063 20838 fs_report.cc:383] FS layout report
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data/data
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.391860 20838 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:50.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.393425 20838 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:50.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.393932 20838 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.396471 20838 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:02:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.401278 20838 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:02:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.401526 20838 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:02:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.401784 20838 ts_tablet_manager.cc:590] Registered 0 tablets
02:02:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.401934 20838 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:02:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.561309 20838 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.130:44319
02:02:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.561393 20968 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.130:44319 every 8 connection(s)
02:02:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.564003 20838 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/data/info.pb
02:02:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.573891 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20838
02:02:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.574568 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-1/wal/instance
02:02:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.588526 20398 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/logs
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data/info.pb
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.131:0
02:02:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.131
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.131
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:02:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.621253 20970 heartbeater.cc:347] Connected to a master server at 127.19.235.189:35125
02:02:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.621838 20970 heartbeater.cc:464] Registering TS with master...
02:02:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.623420 20970 heartbeater.cc:511] Master 127.19.235.189:35125 requested a full tablet report, sending...
02:02:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.627106 20518 ts_manager.cc:194] Registered new tserver with Master: e2060b26df6646bf962abad067f721ef (127.19.235.130:44319)
02:02:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.631283 20969 heartbeater.cc:347] Connected to a master server at 127.19.235.188:42729
02:02:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.631584 20969 heartbeater.cc:464] Registering TS with master...
02:02:50.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.632378 20969 heartbeater.cc:511] Master 127.19.235.188:42729 requested a full tablet report, sending...
02:02:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.634474 20597 ts_manager.cc:194] Registered new tserver with Master: e2060b26df6646bf962abad067f721ef (127.19.235.130:44319)
02:02:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.636082 20597 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.235.130:48997
02:02:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.651733 20971 heartbeater.cc:347] Connected to a master server at 127.19.235.190:36235
02:02:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.652119 20971 heartbeater.cc:464] Registering TS with master...
02:02:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.652766 20971 heartbeater.cc:511] Master 127.19.235.190:36235 requested a full tablet report, sending...
02:02:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:50.655655 20443 ts_manager.cc:194] Registered new tserver with Master: e2060b26df6646bf962abad067f721ef (127.19.235.130:44319)
02:02:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.942548 20978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.943033 20978 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.943475 20978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.974411 20978 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:50.975308 20978 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.235.131
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:51.009111 20978 tablet_server_runner.cc:78] Tablet server non-default flags:
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.235.148:41091
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.235.131:0
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data/info.pb
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.235.131
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.235.190:36235,127.19.235.189:35125,127.19.235.188:42729
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.235.131
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/logs
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:51.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:51.010517 20978 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:51.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:51.012423 20978 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:51.030297 20986 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:51.640091 20969 heartbeater.cc:503] Master 127.19.235.188:42729 was elected leader, sending a full tablet report...
02:02:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:51.030802 20989 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:51.032953 20978 server_base.cc:1034] running on GCE node
02:02:52.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:51.030287 20987 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:52.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.085489 20978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.088199 20978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:52.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.089709 20978 hybrid_clock.cc:648] HybridClock initialized: now 1714615372089613 us; error 61 us; skew 500 ppm
02:02:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.090524 20978 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.097898 20978 webserver.cc:460] Webserver started at http://127.19.235.131:35775/ using document root <none> and password file <none>
02:02:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.098851 20978 fs_manager.cc:359] Metadata directory not provided
02:02:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.099061 20978 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.099462 20978 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.104600 20978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data/instance:
02:02:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32a663503d39461dbc0ead81b3a44b72"
02:02:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:52 on dist-test-slave-66v3"
02:02:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.105907 20978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal/instance:
02:02:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32a663503d39461dbc0ead81b3a44b72"
02:02:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:52 on dist-test-slave-66v3"
02:02:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.114362 20978 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
02:02:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.121246 20996 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.122505 20978 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.122818 20978 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "32a663503d39461dbc0ead81b3a44b72"
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:52 on dist-test-slave-66v3"
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.123147 20978 fs_report.cc:383] FS layout report
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data/data
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:52.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.179955 20978 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:52.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.181478 20978 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.181979 20978 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.184574 20978 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:02:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.189191 20978 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:02:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.189409 20978 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:02:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.189610 20978 ts_tablet_manager.cc:590] Registered 0 tablets
02:02:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.189800 20978 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
02:02:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.333678 20978 rpc_server.cc:307] RPC server started. Bound to: 127.19.235.131:34433
02:02:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.333787 21109 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.235.131:34433 every 8 connection(s)
02:02:52.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.336373 20978 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/data/info.pb
02:02:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.337466 20398 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 20978
02:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.337989 20398 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7152126937674314417/ts-2/wal/instance
02:02:52.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.382105 21110 heartbeater.cc:347] Connected to a master server at 127.19.235.188:42729
02:02:52.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.382846 21110 heartbeater.cc:464] Registering TS with master...
02:02:52.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.384138 21112 heartbeater.cc:347] Connected to a master server at 127.19.235.190:36235
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.384268 21110 heartbeater.cc:511] Master 127.19.235.188:42729 requested a full tablet report, sending...
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.384495 21112 heartbeater.cc:464] Registering TS with master...
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.385178 21111 heartbeater.cc:347] Connected to a master server at 127.19.235.189:35125
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.385470 21112 heartbeater.cc:511] Master 127.19.235.190:36235 requested a full tablet report, sending...
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.385499 21111 heartbeater.cc:464] Registering TS with master...
02:02:52.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.386462 21111 heartbeater.cc:511] Master 127.19.235.189:35125 requested a full tablet report, sending...
02:02:52.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.388485 20597 ts_manager.cc:194] Registered new tserver with Master: 32a663503d39461dbc0ead81b3a44b72 (127.19.235.131:34433)
02:02:52.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.388458 20443 ts_manager.cc:194] Registered new tserver with Master: 32a663503d39461dbc0ead81b3a44b72 (127.19.235.131:34433)
02:02:52.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.389827 20518 ts_manager.cc:194] Registered new tserver with Master: 32a663503d39461dbc0ead81b3a44b72 (127.19.235.131:34433)
02:02:52.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.390069 20597 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.235.131:54503
02:02:52.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:52.398806 20398 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:02:52.410 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:02:52.411 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:02:52.422 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.235.190"
port: 36235
}
bound_http_address {
host: "127.19.235.190"
port: 38333
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.235.189"
port: 35125
}
bound_http_address {
host: "127.19.235.189"
port: 42999
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.235.188"
port: 42729
}
bound_http_address {
host: "127.19.235.188"
port: 37709
}
}
}
02:02:52.427 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:02:52.430 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.235.129"
port: 45301
}
bound_http_address {
host: "127.19.235.129"
port: 38653
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.235.130"
port: 44319
}
bound_http_address {
host: "127.19.235.130"
port: 36841
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.235.131"
port: 34433
}
bound_http_address {
host: "127.19.235.131"
port: 35775
}
}
}
02:02:52.430 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:02:52.608 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:02:52.739 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXxtDsG/test-tmp
02:02:52.739 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:02:52.740 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXxtDsG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:02:52.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:02:53.081 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:02:53.081 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:02:53.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:53.084378 21121 mini_oidc.cc:152] Starting JWKS server
02:02:53.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:53.088827 21121 webserver.cc:460] Webserver started at http://127.0.0.1:33453/ using document root <none> and password file <none>
02:02:53.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:53.089365 21121 mini_oidc.cc:189] Starting OIDC Discovery server
02:02:53.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:53.091542 21121 webserver.cc:460] Webserver started at http://127.0.0.1:43499/ using document root <none> and password file <none>
02:02:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:02:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/principal' for realm 'KRBTEST.COM',
02:02:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:02:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:53 dist-test-slave-66v3 krb5kdc[21133](info): setting up network...
02:02:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:02:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:53 dist-test-slave-66v3 krb5kdc[21133](info): set up 2 sockets
02:02:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:53 dist-test-slave-66v3 krb5kdc[21133](info): commencing operation
02:02:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:53.394668 21110 heartbeater.cc:503] Master 127.19.235.188:42729 was elected leader, sending a full tablet report...
02:02:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.064890 21121 mini_kdc.cc:121] Time spent starting KDC: real 0.935s user 0.000s sys 0.020s
02:02:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:02:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:02:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:02:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:02:54.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:02:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:02:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:02:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:02:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:02:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:02:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/test-user.keytab.
02:02:54.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/test-user.keytab.
02:02:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:54 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:02:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:02:54.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:02:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:02:54.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:02:54Z Disabled control of system clock
02:02:54.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.160.126@KRBTEST.COM; defaulting to no policy
02:02:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.160.126@KRBTEST.COM" created.
02:02:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab.
02:02:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab.
02:02:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:54.242465 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/logs
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.126:42539
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.126
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.126
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:02:54.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.548301 21147 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.548954 21147 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:54.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.549903 21147 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.581843 21147 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:02:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.582104 21147 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.582497 21147 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.582793 21147 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.583082 21147 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.583365 21147 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.583572 21147 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:54.619728 21147 master_runner.cc:386] Master server non-default flags:
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.126
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.126:42539
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:02:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.126
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/logs
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:54.621209 21147 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:54.622402 21147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.640201 21153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:54.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.860996 20628 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8980 after lost signal to thread 20560
02:02:54.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.862107 20628 debug-util.cc:398] Leaking SignalData structure 0x7b080007e640 after lost signal to thread 20631
02:02:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:54.642256 21147 server_base.cc:1034] running on GCE node
02:02:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.644237 21154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:54.641022 21156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.759708 21147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.762984 21147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.764554 21147 hybrid_clock.cc:648] HybridClock initialized: now 1714615375764521 us; error 55 us; skew 500 ppm
02:02:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.765448 21147 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:55 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615375, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:02:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.772868 21147 init.cc:377] Logged in from keytab as kudu/127.20.160.126@KRBTEST.COM (short username kudu)
02:02:55.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.778844 21147 webserver.cc:460] Webserver started at http://127.20.160.126:39983/ using document root <none> and password file <none>
02:02:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.779839 21147 fs_manager.cc:359] Metadata directory not provided
02:02:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.780076 21147 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.780566 21147 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.785549 21147 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/instance:
02:02:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea5d828dc2c74e38a02c6f9996515678"
02:02:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:55 on dist-test-slave-66v3"
02:02:55.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.786799 21147 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal/instance:
02:02:55.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea5d828dc2c74e38a02c6f9996515678"
02:02:55.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:55 on dist-test-slave-66v3"
02:02:55.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.795431 21147 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
02:02:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.801726 21164 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.802915 21147 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:02:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.803236 21147 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:02:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea5d828dc2c74e38a02c6f9996515678"
02:02:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:55 on dist-test-slave-66v3"
02:02:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.803550 21147 fs_report.cc:383] FS layout report
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/data
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.868904 21147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.874786 21147 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.875238 21147 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.978153 21147 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.126:42539
02:02:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.978225 21215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.126:42539 every 8 connection(s)
02:02:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.980913 21147 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:02:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.985522 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21147
02:02:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.986095 21121 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal/instance
02:02:55.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:55.988464 21216 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.003278 21216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } has no permanent_uuid. Determining permanent_uuid...
02:02:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.160.125@KRBTEST.COM; defaulting to no policy
02:02:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.160.125@KRBTEST.COM" created.
02:02:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:56 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615375, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.126@KRBTEST.COM for kudu/127.20.160.126@KRBTEST.COM
02:02:56.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:56.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab.
02:02:56.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab.
02:02:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.043284 21216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } has no permanent_uuid. Determining permanent_uuid...
02:02:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.044255 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:02:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/logs
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.125:35827
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.125
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.125
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:02:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:02:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.045748 21165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.055738 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.105242 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 1
02:02:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.109112 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.177503 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 2
02:02:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.184386 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.275837 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 3
02:02:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.280917 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.424373 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 4
02:02:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.429759 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.433231 21223 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.433920 21223 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.434561 21223 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.467018 21223 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:02:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.467264 21223 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.467525 21223 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.467728 21223 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.467921 21223 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.468117 21223 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.468261 21223 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:02:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.507306 21223 master_runner.cc:386] Master server non-default flags:
02:02:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.125
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.125:35827
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.125
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/logs
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:56.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.508852 21223 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:56.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.510238 21223 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.540810 21232 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:56.729229 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 5
02:02:56.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.738941 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:57.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.286435 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 6
02:02:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:57.324643 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.125:35827: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:02:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:56.542898 21231 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:57.811735 21234 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:57.813958 21233 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1270 milliseconds
02:02:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:57.815548 21223 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.279s user 0.526s sys 0.690s
02:02:57.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:57.815820 21223 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.280s user 0.526s sys 0.690s
02:02:57.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.816099 21223 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:02:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.817251 21223 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:57.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.820678 21223 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:57.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.822240 21223 hybrid_clock.cc:648] HybridClock initialized: now 1714615377822193 us; error 66 us; skew 500 ppm
02:02:57.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.823165 21223 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:57 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615377, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:02:57.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.831543 21223 init.cc:377] Logged in from keytab as kudu/127.20.160.125@KRBTEST.COM (short username kudu)
02:02:57.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.837503 21223 webserver.cc:460] Webserver started at http://127.20.160.125:40481/ using document root <none> and password file <none>
02:02:57.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.838409 21223 fs_manager.cc:359] Metadata directory not provided
02:02:57.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.838606 21223 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:57.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.838982 21223 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.844099 21223 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/instance:
02:02:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec3ffd07ef9949f49ff0059474efa619"
02:02:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:57 on dist-test-slave-66v3"
02:02:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.845227 21223 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal/instance:
02:02:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec3ffd07ef9949f49ff0059474efa619"
02:02:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:57 on dist-test-slave-66v3"
02:02:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.853549 21223 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.001s
02:02:57.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.860297 21247 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:57.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.861528 21223 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.861968 21223 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec3ffd07ef9949f49ff0059474efa619"
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:57 on dist-test-slave-66v3"
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.862309 21223 fs_report.cc:383] FS layout report
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/data
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.936563 21223 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:57.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.941970 21223 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:57.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:57.942425 21223 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:58.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.027544 21223 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.125:35827
02:02:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.027598 21298 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.125:35827 every 8 connection(s)
02:02:58.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.030112 21223 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:02:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.036890 21299 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.038066 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21223
02:02:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.038513 21121 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal/instance
02:02:58.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.160.124@KRBTEST.COM; defaulting to no policy
02:02:58.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.053857 21299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } has no permanent_uuid. Determining permanent_uuid...
02:02:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.160.124@KRBTEST.COM" created.
02:02:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:58 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615377, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.125@KRBTEST.COM for kudu/127.20.160.126@KRBTEST.COM
02:02:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:02:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab.
02:02:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.160.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab.
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.086716 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/logs
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.124:40135
02:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.124
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.124
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:58.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:58.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:58.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:02:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.092229 21299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } has no permanent_uuid. Determining permanent_uuid...
02:02:58.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:58 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615377, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.125@KRBTEST.COM for kudu/127.20.160.125@KRBTEST.COM
02:02:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.119140 21299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } has no permanent_uuid. Determining permanent_uuid...
02:02:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.122082 21251 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.126993 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.176585 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 1
02:02:58.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.181296 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.249709 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 2
02:02:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.253479 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.344911 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 3
02:02:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.349134 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.385172 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } attempt: 7
02:02:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:58 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615375, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.126@KRBTEST.COM for kudu/127.20.160.125@KRBTEST.COM
02:02:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.408993 21216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } has no permanent_uuid. Determining permanent_uuid...
02:02:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.415243 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.416208 21305 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:02:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.416905 21305 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:02:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.418087 21305 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:02:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.450487 21305 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:02:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.450738 21305 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:02:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.451012 21305 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:02:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.451212 21305 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:02:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.451416 21305 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:02:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.451608 21305 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:02:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.451748 21305 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:02:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.473738 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 1
02:02:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.477718 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.488593 21305 master_runner.cc:386] Master server non-default flags:
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.124
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.124:40135
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.124
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/logs
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.489820 21305 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:02:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.490866 21305 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:02:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.492483 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 4
02:02:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.496980 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.510118 21315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.559212 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 2
02:02:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.566049 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.651582 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 3
02:02:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.659013 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.796362 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 5
02:02:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:58.799530 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 4
02:02:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.803280 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.805270 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.088788 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 5
02:02:59.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:59.094400 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.350795 21299 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 6
02:02:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:59.361083 21299 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.511485 21316 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:58.539270 21318 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:02:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:59.627568 21317 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1115 milliseconds
02:02:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.627658 21305 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:02:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.628842 21305 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:02:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.631450 21305 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:02:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.632802 21305 hybrid_clock.cc:648] HybridClock initialized: now 1714615379632773 us; error 37 us; skew 500 ppm
02:02:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.633646 21305 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:02:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615379, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:02:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.639894 21305 init.cc:377] Logged in from keytab as kudu/127.20.160.124@KRBTEST.COM (short username kudu)
02:02:59.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.645566 21305 webserver.cc:460] Webserver started at http://127.20.160.124:42597/ using document root <none> and password file <none>
02:02:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.646693 21305 fs_manager.cc:359] Metadata directory not provided
02:02:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.646895 21305 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:02:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.646943 21216 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } attempt: 6
02:02:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.647444 21305 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:02:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:02:59.653939 21216 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.20.160.124:40135: Network error: Client connection negotiation failed: client connection to 127.20.160.124:40135: connect: Connection refused (error 111)
02:02:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.654500 21305 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/instance:
02:02:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83322921102b47809968e9d2bb21cc2e"
02:02:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:59 on dist-test-slave-66v3"
02:02:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.655742 21305 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal/instance:
02:02:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83322921102b47809968e9d2bb21cc2e"
02:02:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:59 on dist-test-slave-66v3"
02:02:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.664690 21305 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.001s
02:02:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.671972 21331 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:02:59.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.673182 21305 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
02:02:59.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.673569 21305 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:02:59.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83322921102b47809968e9d2bb21cc2e"
02:02:59.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:59 on dist-test-slave-66v3"
02:02:59.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.673949 21305 fs_report.cc:383] FS layout report
02:02:59.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/data
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:02:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:02:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.726001 21305 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:02:59.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.732213 21305 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:02:59.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.732641 21305 kserver.cc:163] Server-wide thread pool size limit: 3276
02:02:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.817273 21305 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.124:40135
02:02:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.817359 21382 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.124:40135 every 8 connection(s)
02:02:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.819900 21305 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:02:59.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.824941 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21305
02:02:59.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.825457 21121 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal/instance
02:02:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.826947 21383 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:02:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.841157 21383 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } has no permanent_uuid. Determining permanent_uuid...
02:02:59.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.160.124@KRBTEST.COM
02:02:59.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615379, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for kudu/127.20.160.126@KRBTEST.COM
02:02:59.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.886096 21383 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } has no permanent_uuid. Determining permanent_uuid...
02:02:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.160.125@KRBTEST.COM
02:02:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615379, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for kudu/127.20.160.125@KRBTEST.COM
02:02:59.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.912675 21383 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } has no permanent_uuid. Determining permanent_uuid...
02:02:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615379, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for kudu/127.20.160.124@KRBTEST.COM
02:02:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.160.126@KRBTEST.COM
02:02:59.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.940014 21121 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
02:02:59.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.940527 21383 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Bootstrap starting.
02:02:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.946178 21383 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Neither blocks nor log segments found. Creating new log.
02:02:59.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:02:59.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:02:59.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.947955 21383 log.cc:824] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Log is configured to *not* fsync() on all Append() calls
02:02:59.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.953083 21383 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: No bootstrap required, opened a new log
02:02:59.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:02:59.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:02:59.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.160.126"
port: 42539
}
bound_http_address {
host: "127.20.160.126"
port: 39983
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.160.125"
port: 35827
}
bound_http_address {
host: "127.20.160.125"
port: 40481
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.160.124"
port: 40135
}
bound_http_address {
host: "127.20.160.124"
port: 42597
}
}
}
02:02:59.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:02:59.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:02:59.964 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:02:59.966 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:02:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.982422 21383 raft_consensus.cc:348] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:02:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.983119 21383 raft_consensus.cc:374] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:02:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.983400 21383 raft_consensus.cc:729] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83322921102b47809968e9d2bb21cc2e, State: Initialized, Role: FOLLOWER
02:02:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.984256 21383 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:02:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.988600 21393 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } } }
02:02:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.990128 21393 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: This master's current role is: FOLLOWER
02:02:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:02:59.990309 21383 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: configured and running, proceeding with master startup.
02:03:00.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:02:59 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:00.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:03:00.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:00.008 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc
02:03:00.016 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:03:00.017 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc
02:03:00.018 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:03:00.022 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:03:00.029 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:03:00.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:03:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.056871 21405 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.057302 21405 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:00.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:00 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615380, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:00.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:03:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.043637 21305 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: OK, instance_id { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" instance_seqno: 1714615375949744 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.073885 21305 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:00.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:00.076 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc
02:03:00.078 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:03:00.079 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.160.126:42539
02:03:00.081 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:03:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.082234 21121 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 21147
02:03:00.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:00.100 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.160.125:35827
02:03:00.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:03:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.101540 21121 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 21223
02:03:00.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:00.121 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.160.124:40135
02:03:00.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:03:00.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.121740 21121 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 21305
02:03:00.139 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:00.139 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.160.126:42539
02:03:00.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.142964 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/logs
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.126:42539
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.126
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39983
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.126
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:03:00.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.443946 21409 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:03:00.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.444515 21409 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.445101 21409 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:03:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.476517 21409 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:03:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.476758 21409 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:03:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.477015 21409 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:03:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.477227 21409 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.477416 21409 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:03:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.477591 21409 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.477751 21409 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.514883 21409 master_runner.cc:386] Master server non-default flags:
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.126.keytab
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.126
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.126:42539
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.126
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39983
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/logs
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.516376 21409 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:00.517591 21409 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:00.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.534981 21417 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:00.535027 21416 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:01.932277 21415 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 21409
02:03:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.036712 21409 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.503s user 0.637s sys 0.842s
02:03:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.037951 21409 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.505s user 0.638s sys 0.842s
02:03:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.038786 21418 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1503 milliseconds
02:03:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.039412 21419 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.039369 21409 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:02.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.042675 21409 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.045110 21409 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:02.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.046478 21409 hybrid_clock.cc:648] HybridClock initialized: now 1714615382046429 us; error 53 us; skew 500 ppm
02:03:02.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.047353 21409 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:02.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:02 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615382, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.053519 21409 init.cc:377] Logged in from keytab as kudu/127.20.160.126@KRBTEST.COM (short username kudu)
02:03:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.058197 21409 webserver.cc:460] Webserver started at http://127.20.160.126:39983/ using document root <none> and password file <none>
02:03:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.059074 21409 fs_manager.cc:359] Metadata directory not provided
02:03:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.059300 21409 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.067061 21409 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
02:03:02.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.071367 21429 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.072491 21409 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.072815 21409 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea5d828dc2c74e38a02c6f9996515678"
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:55 on dist-test-slave-66v3"
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.075206 21409 fs_report.cc:383] FS layout report
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/data
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.124105 21409 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.129706 21409 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.130177 21409 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.197242 21409 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.126:42539
02:03:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.197296 21480 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.126:42539 every 8 connection(s)
02:03:02.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.200068 21409 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/data/info.pb
02:03:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.204174 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21409
02:03:02.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:02.205 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.160.125:35827
02:03:02.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.206749 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/logs
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.125:35827
02:03:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.125
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40481
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.125
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:03:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.213446 21481 sys_catalog.cc:263] Verifying existing consensus state
02:03:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 02:03:02.214211 21481 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:03:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.214586 21409 master.cc:537] Master@127.20.160.126:42539 shutting down...
02:03:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.267097 21409 master.cc:552] Master@127.20.160.126:42539 shutdown complete.
02:03:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:03:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.541716 21483 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:03:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.542500 21483 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.543498 21483 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:03:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.577144 21483 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:03:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.577387 21483 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:03:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.577677 21483 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:03:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.577883 21483 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.578078 21483 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:03:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.578306 21483 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.578454 21483 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:03:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.614478 21483 master_runner.cc:386] Master server non-default flags:
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.125.keytab
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.125
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.125:35827
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.125
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40481
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/logs
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:02.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.615648 21483 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:02.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.616925 21483 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.632213 21491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.632417 21490 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:02.633837 21483 server_base.cc:1034] running on GCE node
02:03:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:02.633033 21493 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.712112 21483 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.714699 21483 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.716114 21483 hybrid_clock.cc:648] HybridClock initialized: now 1714615383716077 us; error 54 us; skew 500 ppm
02:03:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.716919 21483 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:03 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615383, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.722712 21483 init.cc:377] Logged in from keytab as kudu/127.20.160.125@KRBTEST.COM (short username kudu)
02:03:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.727834 21483 webserver.cc:460] Webserver started at http://127.20.160.125:40481/ using document root <none> and password file <none>
02:03:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.728729 21483 fs_manager.cc:359] Metadata directory not provided
02:03:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.728933 21483 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.736917 21483 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
02:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.741397 21502 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.742658 21483 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.742973 21483 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec3ffd07ef9949f49ff0059474efa619"
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:57 on dist-test-slave-66v3"
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.745302 21483 fs_report.cc:383] FS layout report
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/data
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.807245 21483 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.813061 21483 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.813524 21483 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.884721 21483 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.125:35827
02:03:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.884804 21554 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.125:35827 every 8 connection(s)
02:03:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.888551 21483 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/data/info.pb
02:03:03.892 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:03.893 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.160.124:40135
02:03:03.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:03:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.891227 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21483
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.896212 21121 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/logs
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.124:40135
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.124
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42597
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.124
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:03:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:03:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.904145 21555 sys_catalog.cc:263] Verifying existing consensus state
02:03:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 02:03:03.905184 21555 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:03:03.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.909375 21483 master.cc:537] Master@127.20.160.125:35827 shutting down...
02:03:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:03.966537 21483 master.cc:552] Master@127.20.160.125:35827 shutdown complete.
02:03:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:03:04.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.219794 21557 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:03:04.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.220350 21557 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.220950 21557 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.251889 21557 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.252139 21557 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.252405 21557 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.252605 21557 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.252807 21557 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:03:04.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.252987 21557 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:04.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.253131 21557 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:04.290047 21557 master_runner.cc:386] Master server non-default flags:
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.160.84:45021
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/krb5kdc/kudu_127.20.160.124.keytab
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.160.124
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.160.124:40135
02:03:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.160.124
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42597
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/logs
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 01:43:22 UTC on 1a15035a6830
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 84
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:04.291250 21557 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:04.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:04.292219 21557 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:04.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.308300 21566 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.308491 21564 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:04.308919 21563 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:04.309568 21557 server_base.cc:1034] running on GCE node
02:03:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.364645 21557 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.367727 21557 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.369179 21557 hybrid_clock.cc:648] HybridClock initialized: now 1714615385369127 us; error 59 us; skew 500 ppm
02:03:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.370038 21557 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:05 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615385, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.375818 21557 init.cc:377] Logged in from keytab as kudu/127.20.160.124@KRBTEST.COM (short username kudu)
02:03:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.380872 21557 webserver.cc:460] Webserver started at http://127.20.160.124:42597/ using document root <none> and password file <none>
02:03:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.381824 21557 fs_manager.cc:359] Metadata directory not provided
02:03:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.382053 21557 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.393932 21557 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.005s sys 0.000s
02:03:05.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.398417 21574 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:05.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.399536 21557 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:03:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.399894 21557 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data,/tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:03:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83322921102b47809968e9d2bb21cc2e"
02:03:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 02:02:59 on dist-test-slave-66v3"
02:03:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.402366 21557 fs_report.cc:383] FS layout report
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/wal
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/data
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.453073 21557 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.459568 21557 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.460084 21557 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.536006 21557 rpc_server.cc:307] RPC server started. Bound to: 127.20.160.124:40135
02:03:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.536175 21625 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.160.124:40135 every 8 connection(s)
02:03:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.539016 21557 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXxtDsG/test-tmp/mini-kudu-cluster7173682909648048533/master-2/data/info.pb
02:03:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.547575 21121 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu as pid 21557
02:03:05.548 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:03:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.549345 21626 sys_catalog.cc:263] Verifying existing consensus state
02:03:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.555069 21626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Bootstrap starting.
02:03:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.592062 21626 log.cc:824] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Log is configured to *not* fsync() on all Append() calls
02:03:05.593 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.596711 21626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:03:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.597437 21626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Bootstrap complete.
02:03:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.623194 21626 raft_consensus.cc:348] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.626302 21626 raft_consensus.cc:374] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.626610 21626 raft_consensus.cc:729] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83322921102b47809968e9d2bb21cc2e, State: Initialized, Role: FOLLOWER
02:03:05.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.627579 21626 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.630661 21630 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } } }
02:03:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.631839 21630 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: This master's current role is: FOLLOWER
02:03:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.632228 21626 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [sys.catalog]: configured and running, proceeding with master startup.
02:03:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.664394 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:05.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.664705 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.668550 21630 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.669063 21630 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:05.674 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.673981 21630 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.675700 21575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.685727 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:05.690 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.160.124:40135(127.20.160.124:40135)] connecting to peer
02:03:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.695852 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.696837 21578 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.697707 21630 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:05.715427 21580 tablet.cc:2369] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:05.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.668130 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:05.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:05.759714 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:05.763 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:05.763 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:05.764 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:05.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:05.768 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:05.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:05.770 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.160.124:40135(127.20.160.124:40135)] Successfully connected to peer
02:03:05.841 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:03:05.850 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.160.124
02:03:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:06 dist-test-slave-66v3 krb5kdc[21133](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615380, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.160.124@KRBTEST.COM
02:03:06.091 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2e9f2dd0, L:/127.0.0.1:41388 - R:/127.20.160.124:40135] using SASL/GSSAPI
02:03:06.108 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.127 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.152 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.154 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.155 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.155 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.160 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.172 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.174 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.175 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.177 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.180 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.192 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.194 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.195 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.195 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.196 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.212 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.213 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.215 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.215 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.272 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.275 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.276 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.315 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.315 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.316 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.320 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.432 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.434 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.435 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.436 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:06.667902 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:06.668184 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:06.932 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:06.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:06.935 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.935 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.936 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.936 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.937 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:06.937 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:06.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:07.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:07.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:07.435 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:07.435 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:07.436 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:07.436 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:07.436 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:07.436 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:07.439 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:07.671276 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:07.671551 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:07.750474 21651 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:07.750970 21651 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:07.752883 21651 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:07.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:07.759553 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:07.760026 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:07.760409 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:07.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:07.761016 21651 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:08.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:08.674373 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:08.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:08.674630 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:09.132 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:09.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:09.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:09.135 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:09.135 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:09.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:09.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:09.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:09.139 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:09.677433 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:09.677762 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:10.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.298352 21657 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:10.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.298720 21657 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.300082 21657 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:10.305133 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:10.305562 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.305902 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.306421 21657 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:10.680593 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:10.680858 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.762042 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:10.762415 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.683712 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.683976 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:11.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:11.873 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:11.874 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:11.875 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:11.875 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:11.876 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:11.876 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:11.876 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:11.878 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:11.921276 21663 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:11.921638 21663 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:11.923570 21663 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.926503 21578 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.926503 21575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:03:11.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.929441 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:11.929955 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:11.930335 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:11.930917 21663 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:12.686867 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:12.687147 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:13.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:13.690089 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:13.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:13.690379 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:13.963830 21670 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:13.964198 21670 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:13.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:13.965747 21670 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:13.970019 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:13.970448 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:13.970816 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:13.971598 21670 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:14.695559 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:14.695935 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:14.780405 20474 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac5a0 after lost signal to thread 20411
02:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:14.781184 20474 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba560 after lost signal to thread 20477
02:03:14.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:14.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:14.894 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:14.894 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:14.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:14.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:14.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:14.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:14.899 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:15.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:15 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615395, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.379740 21570 init.cc:295] Successfully reacquired a new kerberos TGT
02:03:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.698896 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.699179 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:15.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.721165 20549 debug-util.cc:398] Leaking SignalData structure 0x7b080008f700 after lost signal to thread 20484
02:03:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.721873 20549 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa480 after lost signal to thread 20552
02:03:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.766963 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.767477 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:15.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.880468 21676 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:15.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.880983 21676 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:15.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.882939 21676 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:15.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.899217 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:15.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:15.900617 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:15.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.901019 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:15.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:15.902287 21676 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:16.702194 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:16.702461 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:17.705354 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:17.705726 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:17.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:17.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:17.814 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:17.814 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:17.815 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:17.816 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:17.816 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:17.816 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:17.818 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:18.708621 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:18.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:18.708889 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:19.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:19.712260 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:19.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:19.712527 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:20.715366 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:20.715632 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:20.770148 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:20.770525 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:21.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:21.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:21.382 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:21.382 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:21.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:21.383 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:21.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:21.383 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:21.388 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:21.718503 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:21.718772 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:21.909327 21685 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:21.909771 21685 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:21.911713 21685 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:21.914170 21578 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111) [suppressed 4 similar messages]
02:03:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:21.917074 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:21.917768 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:21.918071 21578 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:21.918609 21685 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:22.721618 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:22.721918 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:23.673769 21691 raft_consensus.cc:482] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:23.674130 21691 raft_consensus.cc:504] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea5d828dc2c74e38a02c6f9996515678" member_type: VOTER last_known_addr { host: "127.20.160.126" port: 42539 } } peers { permanent_uuid: "ec3ffd07ef9949f49ff0059474efa619" member_type: VOTER last_known_addr { host: "127.20.160.125" port: 35827 } } peers { permanent_uuid: "83322921102b47809968e9d2bb21cc2e" member_type: VOTER last_known_addr { host: "127.20.160.124" port: 40135 } }
02:03:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:23.675714 21691 leader_election.cc:288] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539), ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827)
02:03:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:23.680238 21578 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea5d828dc2c74e38a02c6f9996515678 (127.20.160.126:42539): Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111)
02:03:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:23.680702 21575 leader_election.cc:334] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec3ffd07ef9949f49ff0059474efa619 (127.20.160.125:35827): Network error: Client connection negotiation failed: client connection to 127.20.160.125:35827: connect: Connection refused (error 111)
02:03:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:23.681039 21575 leader_election.cc:302] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 83322921102b47809968e9d2bb21cc2e; no voters: ea5d828dc2c74e38a02c6f9996515678, ec3ffd07ef9949f49ff0059474efa619
02:03:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:23.681638 21691 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:03:23.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:23.724958 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:23.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:23.725227 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:24.728790 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:24.729127 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 02:03:25 dist-test-slave-66v3 krb5kdc[21133](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714615405, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:03:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:25.382169 21570 init.cc:295] Successfully reacquired a new kerberos TGT
02:03:25.492 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:25.493 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:25.494 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:25.495 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:25.495 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:25.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:25.496 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:25.496 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:25.498 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:25.732146 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:25.732411 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:25.773644 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:25.774101 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:26.735424 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:26.735687 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:27.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:27.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:27.255 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:27.255 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:27.255 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:27.257 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:27.257 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:27.257 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:27.260 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:27.738401 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:27.738667 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:28.741884 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:28.742271 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:28.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:28.914 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:28.915 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:28.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:28.916 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:28.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:28.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:28.917 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:28.920 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:29.745515 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:29.745827 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:29.873 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:29.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:29.875 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:29.875 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:29.876 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:29.876 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:29.876 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:29.877 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:29.879 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:30.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:30.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:30.254 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:30.254 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:30.254 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:30.255 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:30.255 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:30.255 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:30.257 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:30.748963 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:30.749230 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:30.776796 21557 master_runner.cc:186] Error getting master registration for 127.20.160.126:42539: Network error: Client connection negotiation failed: client connection to 127.20.160.126:42539: connect: Connection refused (error 111),
02:03:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:30.777189 21557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:03:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:31.752732 21641 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83322921102b47809968e9d2bb21cc2e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 02:03:31.753026 21641 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:31.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.126:42539(127.20.160.126:42539)] connecting to peer
02:03:31.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.160.125:35827(127.20.160.125:35827)] connecting to peer
02:03:31.934 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:31.934 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.160.126:42539(127.20.160.126:42539)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:31.934 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.126:42539: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539
02:03:31.935 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:31.935 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.125:35827(127.20.160.125:35827)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:31.935 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.160.125:35827: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
02:03:31.938 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135; will retry
02:03:31.943 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26387), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [210ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [209ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [549ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [553ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [571ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [575ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [577ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [600ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [602ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [604ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [620ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [623ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [625ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [640ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [643ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [645ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [661ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [666ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [721ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [723ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [725ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [760ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [763ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [765ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [880ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [886ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [887ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1384ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1877ms] refreshing cache from master, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3577ms] refreshing cache from master, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [3578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [3579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3582ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [3584ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [3584ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [6317ms] refreshing cache from master, [6317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [6318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [6319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [6320ms] too many traces: truncated at 100 traces, deferred=Deferred@525562510(state=PENDING, result=null, callback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568), errback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568)))
02:03:31.948 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.160.125:35827(127.20.160.125:35827)] disconnecting while in state TERMINATED
02:03:31.949 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.160.124:40135(127.20.160.124:40135)] disconnecting while in state READY
02:03:31.950 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.160.126:42539(127.20.160.126:42539)] disconnecting while in state TERMINATED
02:03:31.955 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:03:31.956 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.160.124:40135(127.20.160.124:40135)] handling channelInactive
02:03:31.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.160.124:40135(127.20.160.124:40135)] cleaning up while in state READY due to: connection closed
02:03:31.958 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:03:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:31.961047 21121 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 21557
02:03:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:03:32Z chronyd exiting
02:03:33.996 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:03:33.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:33.997435 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20679
02:03:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:34.017973 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20838
02:03:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:34.038725 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20978
02:03:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:34.060947 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20410
02:03:34.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:34.088991 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20480
02:03:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 02:03:34.116855 20398 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXxtDsG/build/tsan/bin/kudu with pid 20555
02:03:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T02:03:35Z chronyd exiting
02:03:36.144 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26387), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [210ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [209ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [549ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [553ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [571ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [575ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [577ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [600ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [602ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [604ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [620ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [623ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [625ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [640ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [643ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [645ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [661ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [666ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [721ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [723ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [725ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [760ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [763ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [765ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [880ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [886ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [887ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1384ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1877ms] refreshing cache from master, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3577ms] refreshing cache from master, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [3578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [3579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3582ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [3584ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [3584ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [6317ms] refreshing cache from master, [6317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [6318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [6319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [6320ms] too many traces: truncated at 100 traces, deferred=Deferred@525562510(state=PENDING, result=null, callback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568), errback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.160.125:35827
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=26387), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [210ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [209ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [549ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [553ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [571ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [575ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [577ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [600ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [602ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [604ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [620ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [623ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [625ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [640ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [643ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [645ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [661ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [665ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [666ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [721ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [723ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [725ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [760ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [763ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [765ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [880ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [886ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [887ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1384ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1877ms] refreshing cache from master, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [1880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [1884ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3577ms] refreshing cache from master, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [3578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [3579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.126:42539: Network error: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539, [3582ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.125:35827: Network error: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [3584ms] Sub RPC ConnectToMaster: received response from server master-127.20.160.124:40135: OK, [3584ms] delaying RPC due to: Service unavailable: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827, [6317ms] refreshing cache from master, [6317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.126:42539, [6318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.125:35827, [6319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.160.124:40135, [6320ms] too many traces: truncated at 100 traces, deferred=Deferred@525562510(state=PENDING, result=null, callback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568), errback=(continuation of Deferred@1389142771 after retry RPC after error@1570226526) -> (continuation of Deferred@380788747 after retry RPC after error@317826897) -> (continuation of Deferred@622679238 after retry RPC after error@1189342218) -> (continuation of Deferred@1137999589 after retry RPC after error@1543541473) -> (continuation of Deferred@334776003 after retry RPC after error@1252948747) -> (continuation of Deferred@711792621 after retry RPC after error@1923227611) -> (continuation of Deferred@1137806610 after retry RPC after error@2048835648) -> (continuation of Deferred@948855441 after retry RPC after error@1521548956) -> (continuation of Deferred@356929989 after retry RPC after error@433491949) -> (continuation of Deferred@1376251052 after retry RPC after error@262057070) -> (continuation of Deferred@678501409 after retry RPC after error@99964709) -> (continuation of Deferred@1752873403 after retry RPC after error@310215942) -> (continuation of Deferred@843907964 after retry RPC after error@497952367) -> (continuation of Deferred@539292416 after retry RPC after error@1586436023) -> (continuation of Deferred@67353081 after retry RPC after error@1760721480) -> (continuation of Deferred@1867069605 after retry RPC after error@1239263316) -> (continuation of Deferred@1342704871 after retry RPC after error@735251604) -> (continuation of Deferred@226071899 after retry RPC after error@700580776) -> (continuation of Deferred@960322883 after retry RPC after error@171528540) -> (continuation of Deferred@1261161997 after retry RPC after error@1926056835) -> (continuation of Deferred@558483420 after retry RPC after error@1631244568)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.160.126:42539,127.20.160.125:35827,127.20.160.124:40135) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.126:42539(127.20.160.126:42539): Connection refused: /127.20.160.126:42539,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.160.125:35827(127.20.160.125:35827): Connection refused: /127.20.160.125:35827
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.160.125:35827
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)